Skip to content

Commit 4c373d7

Browse files
adrnadrn
authored
Merge pull request #606 from jeffjennings/requirements
Use individual notebook requirements
2 parents 41b30f4 + 7a68244 commit 4c373d7

46 files changed

Lines changed: 1701 additions & 1509 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/get_modified_tutorials.py‎

Lines changed: 42 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,18 @@
11
from git import Repo
2+
from pathlib import Path
23

34

4-
def main(repo_path, main_branch, **kw):
5+
def get_all_tutorials():
6+
paths = ""
7+
for path in Path("./tutorials").rglob("*.ipynb"):
8+
paths += f" {path}"
9+
print(paths)
10+
11+
12+
def main(repo_path, main_branch, return_all, **kw):
13+
if return_all:
14+
return get_all_tutorials()
15+
516
repo = Repo(repo_path)
617

718
# Check committed changes on this branch against the main branch,
@@ -10,16 +21,20 @@ def main(repo_path, main_branch, **kw):
1021
diff_lists = [
1122
repo.commit(main_branch).diff(repo.head),
1223
repo.head.commit.diff(),
13-
repo.head.commit.diff(None)
24+
repo.head.commit.diff(None),
1425
]
1526

1627
files_changed = set()
1728
for diffs in diff_lists:
18-
files_changed = files_changed.union([
19-
diff.b_path for diff in diffs
20-
if diff.change_type in ['M', 'A', 'R'] # modified, added, renamed
21-
and diff.b_path.endswith('.ipynb')
22-
])
29+
files_changed = files_changed.union(
30+
[
31+
diff.b_path
32+
for diff in diffs
33+
if diff.change_type
34+
in ["M", "A", "R"] # modified, added, renamed
35+
and diff.b_path.endswith(".ipynb")
36+
]
37+
)
2338

2439
if files_changed:
2540
print(" ".join(files_changed))
@@ -30,17 +45,28 @@ def main(repo_path, main_branch, **kw):
3045

3146
parser = ArgumentParser()
3247
parser.add_argument(
33-
'-r', '--repo-path',
34-
dest='repo_path',
35-
default='.',
36-
help='The path to the root of the astropy-tutorials repository folder.'
48+
"-r",
49+
"--repo-path",
50+
dest="repo_path",
51+
default=".",
52+
help="The path to the root of the astropy-tutorials repository folder.",
53+
)
54+
parser.add_argument(
55+
"--main-branch",
56+
dest="main_branch",
57+
default="main",
58+
help=(
59+
"The name of the main branch to compare against. Default is "
60+
'"main" but on CI it should be origin/main.'
61+
),
3762
)
3863
parser.add_argument(
39-
'--main-branch',
40-
dest='main_branch',
41-
default='main',
42-
help=('The name of the main branch to compare against. Default is '
43-
'"main" but on CI it should be origin/main.')
64+
"--return_all",
65+
dest="return_all",
66+
default=False,
67+
help=(
68+
"Whether to return all tutorials rather than just those modified."
69+
),
4470
)
4571
args = parser.parse_args()
4672
main(**vars(args))

‎.github/workflows/build.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,12 @@ jobs:
3030
- name: Execute the notebooks
3131
run: |
3232
make executeall
33+
shell: bash
3334

3435
- name: Convert the notebooks to HTML
3536
run: |
3637
make convertall
38+
shell: bash
3739

3840
- name: Name artifact
3941
id: nameartifact

‎.github/workflows/prs.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,16 +36,17 @@ jobs:
3636
if: contains(github.event.pull_request.labels.*.name, 'Run all tutorials')
3737
run: |
3838
export TUTORIALS_MAIN_BRANCH=origin/main
39-
make executeall
40-
make convertall
39+
make clean
40+
make buildall
41+
shell: bash
4142

4243
# Otherwise, only run tutorials that have been modified
4344
- name: Run only modified tutorials
4445
if: "!contains(github.event.pull_request.labels.*.name, 'Run all tutorials')"
4546
run: |
4647
export TUTORIALS_MAIN_BRANCH=origin/main
47-
make execute
48-
make convert
48+
make build
49+
shell: bash
4950

5051
- name: Name artifact
5152
id: nameartifact

‎Makefile‎

Lines changed: 45 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,65 @@
11
default: build
22

3+
SHELL := /bin/bash
4+
35
TUTORIALS_MAIN_BRANCH ?= main
4-
MODIFIED := $(shell python .github/get_modified_tutorials.py --main-branch $(TUTORIALS_MAIN_BRANCH))
6+
7+
# paths to the individual notebooks that have been modified
8+
MODIFIED_NOTEBOOKS := $(shell python .github/get_modified_tutorials.py --main-branch $(TUTORIALS_MAIN_BRANCH))
9+
# paths to the individual requirements.txt files in the directories in which 1+ notebooks have been modified
10+
MODIFIED_RQT_PATHS := $(foreach var,$(MODIFIED_NOTEBOOKS),$(addsuffix requirements.txt,$(dir $(var))))
11+
12+
ALL_NOTEBOOKS := $(shell python .github/get_modified_tutorials.py --main-branch $(TUTORIALS_MAIN_BRANCH) --return_all true)
13+
ALL_RQT_PATHS := $(foreach var,$(ALL_NOTEBOOKS),$(addsuffix requirements.txt,$(dir $(var))))
514

615
FLAGS = --flatten --build-path=. -v
7-
CONVERTFLAGS = --make-index --preprocessors=nbconvert.preprocessors.ExtractOutputPreprocessor --index-template=templates/index.tpl
16+
CONVERTFLAGS = --make-index --preprocessors=nbconvert.preprocessors.ExtractOutputPreprocessor --index-template=templates/index.tpl --overwrite
817

918
init:
1019
python -m pip install -U -r requirements-dev.txt
11-
pre-commit install
12-
13-
build: envcheck execute convert
14-
buildall: envcheck executeall convertall
1520

16-
envcheck:
17-
python -c "import pkg_resources; pkg_resources.require(open('requirements.txt', mode='r')); print('Your environment is all set!')"
21+
build: convert
22+
buildall: convertall
1823

1924
execute:
20-
nbcollection execute --timeout=600 ${FLAGS} ${MODIFIED}
25+
i=0; \
26+
_paths=($(MODIFIED_RQT_PATHS)); \
27+
for notebook in ${MODIFIED_NOTEBOOKS}; do \
28+
echo Installing requirements from $${_paths[i]}; \
29+
python -m pip install --force-reinstall -r $${_paths[i]} > /dev/null; \
30+
nbcollection execute --timeout=600 ${FLAGS} $$notebook; \
31+
i=$$((i+1)); \
32+
done
2133

2234
convert:
23-
nbcollection convert ${CONVERTFLAGS} ${FLAGS} ${MODIFIED}
35+
i=0; \
36+
_paths=($(MODIFIED_RQT_PATHS)); \
37+
for notebook in ${MODIFIED_NOTEBOOKS}; do \
38+
echo Installing requirements from $${_paths[i]}; \
39+
python -m pip install --force-reinstall -r $${_paths[i]} > /dev/null; \
40+
nbcollection convert ${CONVERTFLAGS} ${FLAGS} $$notebook; \
41+
i=$$((i+1)); \
42+
done
2443

2544
executeall:
26-
nbcollection execute --timeout=600 ${FLAGS} tutorials
45+
i=0; \
46+
_paths=(${ALL_RQT_PATHS}); \
47+
for notebook in ${ALL_NOTEBOOKS}; do \
48+
echo Installing requirements from $${_paths[i]}; \
49+
python -m pip install --force-reinstall -r $${_paths[i]} > /dev/null; \
50+
nbcollection execute --timeout=600 ${FLAGS} $$notebook; \
51+
i=$$((i+1)); \
52+
done
2753

2854
convertall:
29-
nbcollection convert ${CONVERTFLAGS} ${FLAGS} tutorials
55+
i=0; \
56+
_paths=($(ALL_RQT_PATHS)); \
57+
for notebook in ${ALL_NOTEBOOKS}; do \
58+
echo Installing requirements from $${_paths[i]}; \
59+
python -m pip install --force-reinstall -r $${_paths[i]} > /dev/null; \
60+
nbcollection convert ${CONVERTFLAGS} ${FLAGS} $$notebook; \
61+
i=$$((i+1)); \
62+
done
3063

3164
clean:
3265
rm -rf _build

‎requirements-dev.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
-r requirements.txt
1+
# Note: these are the requirements *in addition to* those listed in the unique
2+
# requirements for each set of notebooks - see astropy-tutorials/tutorials/*/requirements.txt
23
gitpython
34
git+https://github.com/astropy/nbcollection
45
pre-commit
6+
ipykernel

‎requirements.txt‎

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎tutorials/FITS-cubes/FITS-cubes.ipynb‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,25 +27,25 @@
2727
"\n",
2828
"The primary libraries we'll be using are: [astroquery](http://www.astropy.org/astroquery/), [spectral_cube](https://spectral-cube.readthedocs.io/en/latest/), [reproject](https://reproject.readthedocs.io/en/stable/#), [matplotlib](https://matplotlib.org/)) \n",
2929
"\n",
30-
"They can be installed using conda:"
31-
]
32-
},
33-
{
34-
"cell_type": "markdown",
35-
"metadata": {},
36-
"source": [
30+
"They can be installed using conda:\n",
31+
"\n",
3732
"```\n",
3833
"conda install -c conda-forge astroquery\n",
3934
"conda install -c conda-forge spectral-cube\n",
4035
"conda install -c conda-forge reproject\n",
41-
"```"
36+
"```\n",
37+
"\n",
38+
"Alternatively, if you don't use conda, you can use pip."
4239
]
4340
},
4441
{
45-
"cell_type": "markdown",
42+
"cell_type": "code",
43+
"execution_count": null,
4644
"metadata": {},
45+
"outputs": [],
4746
"source": [
48-
"Alternatively, if you don't use conda, you can use pip."
47+
"with open('requirements.txt') as f:\n",
48+
" print(f\"Required packages for this notebook:\\n{f.read()}\")"
4949
]
5050
},
5151
{

‎tutorials/FITS-cubes/requirements.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
astropy
2-
astroquery
2+
astroquery>=0.4.8.dev9474 # 2024-09-24 pinned for Gaia column capitalization issue
33
matplotlib
44
numpy
55
reproject
6-
spectral_cube
6+
spectral_cube @ git+https://github.com/radio-astro-tools/spectral-cube # as of: 2024-10-10 for issue with 'distutils'

‎tutorials/FITS-header/FITS-header.ipynb‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,16 @@
2424
"keyword."
2525
]
2626
},
27+
{
28+
"cell_type": "code",
29+
"execution_count": null,
30+
"metadata": {},
31+
"outputs": [],
32+
"source": [
33+
"with open('requirements.txt') as f:\n",
34+
" print(f\"Required packages for this notebook:\\n{f.read()}\")"
35+
]
36+
},
2737
{
2838
"cell_type": "code",
2939
"execution_count": null,

‎tutorials/FITS-header/input_file.fits‎

0 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)