diff --git a/.github/docker/Dockerfile b/.github/docker/Dockerfile new file mode 100644 index 0000000..fee61bb --- /dev/null +++ b/.github/docker/Dockerfile @@ -0,0 +1,51 @@ +# syntax=docker/dockerfile:1 +# Environment image for DANDI example notebooks. Built by +# .github/workflows/build-notebook-images.yml from a context prepared by +# .github/scripts/build_notebook_image.py (requirements.txt + work/). +# +# Two separate Python environments by design: +# - the kernel env is the system Python (/usr/local): pins go in with +# `uv pip install --system`, matching both the Colab bootstrap cell and +# run_notebook.py, so the CI harness runs unmodified inside the image; +# - JupyterLab lives in an isolated `uv tool` env so its own dependency +# tree can never upgrade or downgrade anything in the pinned kernel env. + +ARG BASE_IMAGE=python:3.12-slim@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a +FROM ${BASE_IMAGE} + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:0.12.5 /uv /uvx /usr/local/bin/ + +ARG JUPYTERLAB_VERSION=4.6.3 +ENV UV_TOOL_BIN_DIR=/opt/uv-bin +RUN uv tool install "jupyterlab==${JUPYTERLAB_VERSION}" +# JUPYTER_PATH makes the pinned kernelspec under /usr/local win over the +# kernelspec that ships inside the JupyterLab tool env. +ENV PATH="/opt/uv-bin:${PATH}" \ + JUPYTER_PATH=/usr/local/share/jupyter + +# ipykernel and nbformat resolve jointly with the pins in one invocation so +# any conflict fails the build instead of silently changing a pinned version. +COPY requirements.txt /tmp/requirements.txt +RUN uv pip install --system --no-cache -r /tmp/requirements.txt ipykernel nbformat \ + && python -m ipykernel install --name python3 --display-name "Python 3 (pinned)" + +WORKDIR /work +COPY work/ /work/ + +ARG DEFAULT_NOTEBOOK +ARG BUILD_HASH +ARG GIT_SHA=unknown +ARG NOTEBOOKS="" +ENV DEFAULT_NOTEBOOK=${DEFAULT_NOTEBOOK} +LABEL org.opencontainers.image.source="https://github.com/dandi/example-notebooks" \ + org.opencontainers.image.revision="${GIT_SHA}" \ + org.dandiarchive.notebooks="${NOTEBOOKS}" \ + org.dandiarchive.build-hash="${BUILD_HASH}" + +EXPOSE 8888 +CMD jupyter-lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root \ + --ServerApp.default_url="/lab/tree/${DEFAULT_NOTEBOOK}" diff --git a/.github/docker/README.md b/.github/docker/README.md new file mode 100644 index 0000000..5544cbc --- /dev/null +++ b/.github/docker/README.md @@ -0,0 +1,78 @@ +# Containerized Notebook Environments + +Each Colab-ready notebook in this repository is also published as a +self-contained container image on the GitHub Container Registry. The image +bundles the notebook, any helper files it fetches, a JupyterLab server, and +the exact pinned dependency set from the notebook's install cell, +preinstalled. Unlike the Colab path, the image does not depend on Colab's +current Python version or on PyPI still serving the pinned packages, so it +remains runnable long after the hosted environments have moved on. + +## Running a Notebook Image + +``` +docker run --rm -p 8888:8888 ghcr.io/dandi/example-notebooks/001550-paganlab:latest +``` + +Then open the `http://127.0.0.1:8888/lab?token=...` URL printed in the +terminal. JupyterLab opens on the notebook with its dependencies already +installed; the install cell at the top is a no-op and can be skipped. The +notebooks stream data from the DANDI Archive, so network access is still +required at run time. + +Images are built for `linux/amd64`, the platform the dependency pins were +resolved for. On Apple Silicon, Docker Desktop runs them under emulation; +pass `--platform linux/amd64` to silence the platform warning. + +Images are named after the notebook directory (lowercased, with `/` replaced +by `-`). When notebooks in the same directory pin different dependency sets, +the notebook name is appended, e.g. +`ghcr.io/dandi/example-notebooks/000971-lernerlab-seiler-2024-optogenetics-example-notebook`. + +## Tags + +| Tag | Meaning | +| --- | --- | +| `latest` | most recent verified build | +| `YYYY-MM-DD` | date-stamped snapshot, useful as a citable reference | +| `sha-` | git commit the image was built from | +| `hash-<12 hex>` | content hash of the build inputs (pins, notebooks, helpers, Dockerfile); used by CI to skip unchanged rebuilds | + +## How Images Are Built and Verified + +The `Build notebook images` workflow +(`.github/workflows/build-notebook-images.yml`) groups notebooks by directory +and pin set (`.github/scripts/build_notebook_image.py`), builds one image per +group from the parameterized `Dockerfile` in this directory, and then runs +every notebook in the group **inside the candidate image** using the same +harness the test workflows use (`.github/scripts/run_notebook.py`). An image +is pushed only if all of its notebooks execute successfully, so `latest` is +always a verified snapshot. + +Inside the image the kernel environment is the system Python, with pins +installed via `uv pip install --system`, exactly matching the Colab bootstrap +cell and the CI harness. JupyterLab runs from an isolated `uv tool` +environment so its own dependencies cannot perturb the pinned kernel +environment. `ipykernel` and `nbformat` are the only additions to the pinned +set; they resolve jointly with the pins so a conflict fails the build rather +than silently changing a pinned version. + +## Maintainer Notes + +- The workflow is currently `workflow_dispatch` only. The `filter` input is a + regex on the notebook directory or path (default: the `001550/PaganLab` + pilot); `push: false` gives a dry run (build + verify, no publish); `force` + rebuilds even when the build-hash check says the published image is current. +- Rebuilds are skipped when a `hash-<...>` tag matching the current build + inputs already exists on the registry. Changing a notebook, its pins, its + helpers, or the Dockerfile changes the hash and triggers a rebuild. Pins are + refreshed with `.github/scripts/lock_notebook.py` from the `requirements.in` + committed next to the notebook (see `docs/adding-notebooks.md`). +- The base image is digest-pinned in the Dockerfile (`ARG BASE_IMAGE`). To + bump it, update the digest, which changes every group's build hash, and + dispatch a full rebuild. The `BASE_IMAGE` arg is also the knob for a future + variant based on the official Colab runtime image. +- Images run as root (`python:3.12-slim` has no unprivileged user) and keep + Jupyter's token auth enabled. The published port binding in the docs is + loopback-only via `-p 8888:8888` on a local machine; advise users not to + bind on public interfaces. diff --git a/.github/scripts/build_notebook_image.py b/.github/scripts/build_notebook_image.py new file mode 100644 index 0000000..42b6441 --- /dev/null +++ b/.github/scripts/build_notebook_image.py @@ -0,0 +1,233 @@ +"""Group notebooks into container-image build units and prepare build contexts. + +Notebooks that carry the Colab-bootstrap install cell are grouped by +(directory, pin-set): notebooks in the same directory whose install cells pin +the identical dependency set share one image. The image for a group contains +every file of that directory (minus notebooks belonging to other groups) plus +any helper files the install cells fetch via `!curl`/`!wget`, with the pinned +dependencies preinstalled into the system Python. + +Subcommands: + list-groups [--filter REGEX] [--names-only] + Print the groups as JSON. `--names-only` emits just the group names, + suitable for a GitHub Actions matrix. + prepare --name NAME --context-dir DIR [--image-prefix PREFIX] + Write a docker build context (requirements.txt + work/) for one group + and emit image name, build hash, default notebook, and the notebook + list to $GITHUB_OUTPUT (or stdout when unset). + +The pin extraction reuses `find_install_cell` from run_notebook.py, so the +image contents stay in lockstep with what CI tests. Entries captured by that +regex that are not actual pins (it also picks up the literal `form` from the +`{ display-mode: "form" }` title line) are filtered out: only `pkg==ver` and +`pkg @ url` requirements are baked into an image. + +Assumes `nbformat` is importable. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +import nbformat + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from list_notebooks import REPO_ROOT, is_excluded, load_exclusions # noqa: E402 +from run_notebook import find_install_cell # noqa: E402 + +DOCKERFILE = REPO_ROOT / ".github" / "docker" / "Dockerfile" +DEFAULT_IMAGE_PREFIX = "ghcr.io/dandi/example-notebooks" + + +def slug(s: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") + + +def real_pins(pins: list[str]) -> tuple[list[str], list[str]]: + """Split install-cell entries into actual requirements and strays.""" + kept = [p for p in pins if "==" in p or " @ " in p] + dropped = [p for p in pins if p not in kept] + return kept, dropped + + +@dataclass +class Group: + directory: str # repo-relative, e.g. "001550/PaganLab" + pin_hash: str + pins: list[str] + helpers: list[str] = field(default_factory=list) + notebooks: list[str] = field(default_factory=list) # filenames within directory + name: str = "" + + def as_dict(self) -> dict: + return { + "name": self.name, + "directory": self.directory, + "pin_hash": self.pin_hash, + "notebooks": sorted(self.notebooks), + } + + +def collect_groups() -> list[Group]: + exclusions = load_exclusions() + groups: dict[tuple[str, str], Group] = {} + for path in sorted(REPO_ROOT.rglob("*.ipynb")): + if ".ipynb_checkpoints" in path.parts: + continue + rel = str(path.relative_to(REPO_ROOT)) + if is_excluded(rel, exclusions): + continue + nb = nbformat.read(path, as_version=4) + try: + pins, helpers, _ = find_install_cell(nb) + except RuntimeError: + continue # no Colab bootstrap -> no image + kept, _ = real_pins(pins) + pin_hash = hashlib.sha256("\n".join(sorted(kept)).encode()).hexdigest() + directory = str(path.parent.relative_to(REPO_ROOT)) + key = (directory, pin_hash) + group = groups.setdefault( + key, Group(directory=directory, pin_hash=pin_hash, pins=sorted(kept)) + ) + group.notebooks.append(path.name) + for h in helpers: + if h not in group.helpers: + group.helpers.append(h) + + per_dir: dict[str, list[Group]] = {} + for g in groups.values(): + per_dir.setdefault(g.directory, []).append(g) + for dir_groups in per_dir.values(): + for g in dir_groups: + g.name = slug(g.directory) + if len(dir_groups) > 1: + g.name += "-" + slug(Path(sorted(g.notebooks)[0]).stem) + + result = sorted(groups.values(), key=lambda g: g.name) + names = [g.name for g in result] + assert len(names) == len(set(names)), f"group name collision: {names}" + return result + + +def matches(group: Group, pattern: str) -> bool: + if not pattern: + return True + rx = re.compile(pattern) + return bool( + rx.search(group.directory) + or any(rx.search(f"{group.directory}/{n}") for n in group.notebooks) + ) + + +def cmd_list_groups(args: argparse.Namespace) -> int: + groups = [g for g in collect_groups() if matches(g, args.filter)] + if args.names_only: + print(json.dumps([g.name for g in groups])) + else: + print(json.dumps([g.as_dict() for g in groups], indent=2)) + return 0 + + +def cmd_prepare(args: argparse.Namespace) -> int: + groups = [g for g in collect_groups() if g.name == args.name] + if not groups: + print(f"error: no group named {args.name!r}", file=sys.stderr) + return 1 + group = groups[0] + + context = Path(args.context_dir) + if context.exists(): + shutil.rmtree(context) + work = context / "work" + + group_notebooks = set(group.notebooks) + src_dir = REPO_ROOT / group.directory + + def ignore(directory: str, names: list[str]) -> list[str]: + ignored = [n for n in names if n == ".ipynb_checkpoints"] + for n in names: + if not n.endswith(".ipynb"): + continue + in_group = ( + Path(directory) == src_dir and n in group_notebooks + ) + if not in_group: + ignored.append(n) + return ignored + + shutil.copytree(src_dir, work, ignore=ignore) + + (context / "requirements.txt").write_text("\n".join(group.pins) + "\n") + + # Bake helper files the same way run_notebook.py fetches them in CI. + for h in group.helpers: + cmd_str = h.lstrip("!").strip() + m = re.search(r"-o\s+(\S+)", cmd_str) + if m: + target = work / m.group(1) + target.parent.mkdir(parents=True, exist_ok=True) + r = subprocess.run( + ["bash", "-c", cmd_str], cwd=str(work), capture_output=True, text=True + ) + if r.returncode != 0: + print(f"error: helper failed: {cmd_str}\n{r.stderr[-1000:]}", file=sys.stderr) + return 1 + + h = hashlib.sha256() + for pin in group.pins: + h.update(pin.encode() + b"\n") + for helper in group.helpers: + h.update(helper.encode() + b"\n") + for name in sorted(group.notebooks): + h.update((REPO_ROOT / group.directory / name).read_bytes()) + h.update(DOCKERFILE.read_bytes()) + build_hash = h.hexdigest() + + notebooks = sorted(group.notebooks) + outputs = { + "image": f"{args.image_prefix}/{group.name}", + "build_hash": build_hash, + "build_hash_short": build_hash[:12], + "default_notebook": notebooks[0], + "notebooks": json.dumps(notebooks), + "context_dir": str(context), + } + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with open(github_output, "a") as f: + for k, v in outputs.items(): + f.write(f"{k}={v}\n") + print(json.dumps(outputs, indent=2)) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + p_list = sub.add_parser("list-groups") + p_list.add_argument("--filter", default="", help="regex on directory or notebook path") + p_list.add_argument("--names-only", action="store_true") + p_list.set_defaults(func=cmd_list_groups) + + p_prep = sub.add_parser("prepare") + p_prep.add_argument("--name", required=True) + p_prep.add_argument("--context-dir", required=True) + p_prep.add_argument("--image-prefix", default=DEFAULT_IMAGE_PREFIX) + p_prep.set_defaults(func=cmd_prepare) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/lock_notebook.py b/.github/scripts/lock_notebook.py new file mode 100644 index 0000000..6531a9a --- /dev/null +++ b/.github/scripts/lock_notebook.py @@ -0,0 +1,158 @@ +"""Generate or refresh a notebook's Colab-bootstrap cells from a requirements.in. + +Contributors commit a `requirements.in` next to their notebook listing only the +notebook's direct dependencies (e.g. `pynwb`, `remfile`, `matplotlib`). This +script compiles that into a fully pinned set with `uv pip compile`, constrained +to Colab's preinstalled versions, and writes the four bootstrap cells (badge, +install intro, pinned install cell, restart admonition) into the notebook — +prepending them when absent, or refreshing the install cell's pin block in +place (helper `!curl`/`!wget` lines are preserved) when already present. + +The requirements file is resolved per notebook: `.requirements.in` next +to the notebook wins when present, otherwise the directory's `requirements.in` +is used. One `requirements.in` shared by all notebooks in a directory keeps +their pin sets identical, so they are tested against one environment and are +published together in one container image (see ../docker/README.md). + +Usage: + python .github/scripts/lock_notebook.py [...] + +Assumes `uv` is on PATH and `nbformat` is importable. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +import nbformat + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from list_notebooks import REPO_ROOT # noqa: E402 +from run_notebook import find_install_cell # noqa: E402 + +CONSTRAINT = REPO_ROOT / ".github" / "colab-preinstalled.txt" + +BADGE = ( + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)]" + "(https://colab.research.google.com/github/dandi/example-notebooks/blob/master/{path})" +) +INTRO = ( + "## Installing requirements\n" + "\n" + "The cell below installs every Python package needed to run this notebook, " + "at fully pinned versions, using [`uv`](https://github.com/astral-sh/uv) for " + "fast resolution. In Colab the cell is collapsed by default — click the " + "▶ button to run it." +) +RESTART = ( + "> **⚠️ Restart runtime after install**\n" + ">\n" + "> The install may upgrade packages already loaded in the kernel. Go to " + "**Runtime → Restart session**, then **Run all cells below** (skip this " + "install cell on re-run)." +) +INSTALL_HEADER = ( + '#@title Installing requirements (click ▶ to run) { display-mode: "form" }\n' + "# Colab provides Python 3.12. We install with `uv --system` because Colab's\n" + "# kernel runs outside a virtualenv. All versions (direct + transitive) are\n" + "# pinned below so the notebook is reproducible regardless of resolver drift.\n" + "!pip install -q uv\n" +) + + +def requirements_for(nb_path: Path) -> Path: + per_notebook = nb_path.with_name(f"{nb_path.stem}.requirements.in") + if per_notebook.exists(): + return per_notebook + shared = nb_path.parent / "requirements.in" + if shared.exists(): + return shared + raise FileNotFoundError( + f"No requirements file for {nb_path}: expected {per_notebook.name} or " + f"requirements.in in {nb_path.parent}/. List the notebook's direct " + "dependencies there (one per line), then re-run this script." + ) + + +def compile_pins(requirements: Path) -> list[str]: + cmd = [ + "uv", "pip", "compile", str(requirements), + "--python-version", "3.12", + "--python-platform", "linux", + "--constraint", str(CONSTRAINT), + "--no-header", "--no-annotate", + ] + r = subprocess.run(cmd, capture_output=True, text=True, stdin=subprocess.DEVNULL) + if r.returncode != 0: + raise RuntimeError(f"uv pip compile failed for {requirements}:\n{r.stderr}") + pins = [ + line.strip() + for line in r.stdout.splitlines() + if line.strip() and not line.strip().startswith("#") + ] + if not pins: + raise RuntimeError(f"uv pip compile produced no pins from {requirements}") + return pins + + +def install_cell_source(pins: list[str], helpers: list[str]) -> str: + lines = [INSTALL_HEADER + "!uv pip install --system \\"] + lines += [f' "{pin}" \\' for pin in pins[:-1]] + lines.append(f' "{pins[-1]}"') + lines += helpers + return "\n".join(lines) + + +def lock(nb_path: Path) -> None: + requirements = requirements_for(nb_path) + pins = compile_pins(requirements) + nb = nbformat.read(nb_path, as_version=4) + + try: + _, helpers, install_idx = find_install_cell(nb) + except RuntimeError: + helpers, install_idx = [], None + + if install_idx is not None: + nb.cells[install_idx].source = install_cell_source(pins, helpers) + nb.cells[install_idx].metadata["cellView"] = "form" + action = "refreshed install cell in" + else: + rel = nb_path.resolve().relative_to(REPO_ROOT) + install_cell = nbformat.v4.new_code_cell(install_cell_source(pins, helpers)) + install_cell.metadata["cellView"] = "form" + nb.cells = [ + nbformat.v4.new_markdown_cell(BADGE.format(path=str(rel).replace(" ", "%20"))), + nbformat.v4.new_markdown_cell(INTRO), + install_cell, + nbformat.v4.new_markdown_cell(RESTART), + ] + nb.cells + # Cell ids require nbformat 4.5; prepended cells carry ids. + nb.nbformat_minor = max(nb.nbformat_minor, 5) + action = "prepended bootstrap cells to" + + nbformat.validate(nb) + nbformat.write(nb, nb_path) + print(f"{action} {nb_path} ({len(pins)} pins from {requirements.name}" + + (f", kept {len(helpers)} helper line(s)" if helpers else "") + ")") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("notebooks", nargs="+", type=Path) + args = parser.parse_args() + failures = 0 + for nb_path in args.notebooks: + try: + lock(nb_path) + except Exception as e: + print(f"error: {nb_path}: {e}", file=sys.stderr) + failures += 1 + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/run_notebook.py b/.github/scripts/run_notebook.py index 27bd88c..94b66ee 100644 --- a/.github/scripts/run_notebook.py +++ b/.github/scripts/run_notebook.py @@ -56,7 +56,9 @@ def find_install_cell(nb): "CI walks every notebook in the repo and expects each one to begin with a " "code cell containing `!uv pip install --system \"pkg==ver\" ...` so that " "its dep set is fully pinned and reproducible. " - "To fix: add the bootstrap cells (see PR #149 for the pattern) — OR, if " + "To fix: commit a `requirements.in` next to the notebook and run " + "`python .github/scripts/lock_notebook.py ` to generate the " + "bootstrap cells (see docs/adding-notebooks.md) — OR, if " "the notebook genuinely can't be tested headlessly (needs a database, " "proprietary creds, etc), add its path to `.github/notebook-test-exclusions.txt` " "with a comment explaining why." diff --git a/.github/workflows/build-notebook-images.yml b/.github/workflows/build-notebook-images.yml new file mode 100644 index 0000000..08ef2e9 --- /dev/null +++ b/.github/workflows/build-notebook-images.yml @@ -0,0 +1,151 @@ +name: Build notebook images + +# Builds a container image per notebook group (directory + identical pin set), +# verifies every notebook in the group runs inside the candidate image using +# the same harness as the test workflows, and pushes verified images to +# ghcr.io. Only verified images are ever published. + +on: + workflow_dispatch: + inputs: + filter: + description: "Regex; build only groups whose directory or notebook path matches" + default: "^001550/PaganLab" + force: + description: "Rebuild and push even if an image with this build hash exists" + type: boolean + default: false + push: + description: "Push to ghcr.io (uncheck for a dry-run build + verify)" + type: boolean + default: true + +permissions: + contents: read + packages: write + +jobs: + list: + runs-on: ubuntu-latest + outputs: + groups: ${{ steps.groups.outputs.groups }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install runner dependencies + run: pip install --no-cache-dir nbformat + + - name: List image groups + id: groups + run: | + python .github/scripts/build_notebook_image.py list-groups \ + --filter '${{ github.event.inputs.filter }}' --names-only > groups.json + echo "groups=$(cat groups.json)" >> "$GITHUB_OUTPUT" + echo "Matched groups:" + cat groups.json + + build: + needs: list + if: needs.list.outputs.groups != '[]' + runs-on: ubuntu-latest + timeout-minutes: 90 + strategy: + fail-fast: false + max-parallel: 6 + matrix: + group: ${{ fromJSON(needs.list.outputs.groups) }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install runner dependencies + run: pip install --no-cache-dir nbformat + + - name: Prepare build context + id: prepare + run: | + python .github/scripts/build_notebook_image.py prepare \ + --name '${{ matrix.group }}' \ + --context-dir "$RUNNER_TEMP/context" \ + --image-prefix "ghcr.io/${{ github.repository }}" + + - name: Log in to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Skip if an image with this build hash is already published + id: check + run: | + IMAGE='${{ steps.prepare.outputs.image }}' + TAG='hash-${{ steps.prepare.outputs.build_hash_short }}' + if [ '${{ inputs.force }}' != 'true' ] \ + && docker manifest inspect "$IMAGE:$TAG" >/dev/null 2>&1; then + echo "Image $IMAGE:$TAG already exists; skipping build and push." + echo "up_to_date=true" >> "$GITHUB_OUTPUT" + else + echo "up_to_date=false" >> "$GITHUB_OUTPUT" + fi + + - name: Build candidate image + if: steps.check.outputs.up_to_date == 'false' + run: | + docker build \ + --platform linux/amd64 \ + -f .github/docker/Dockerfile \ + --build-arg DEFAULT_NOTEBOOK='${{ steps.prepare.outputs.default_notebook }}' \ + --build-arg BUILD_HASH='${{ steps.prepare.outputs.build_hash }}' \ + --build-arg GIT_SHA='${{ github.sha }}' \ + --build-arg NOTEBOOKS='${{ steps.prepare.outputs.notebooks }}' \ + -t '${{ steps.prepare.outputs.image }}:candidate' \ + "$RUNNER_TEMP/context" + + - name: Verify notebooks inside the candidate image + if: steps.check.outputs.up_to_date == 'false' + run: | + mkdir -p "$RUNNER_TEMP/verify" + echo '${{ steps.prepare.outputs.notebooks }}' \ + | python3 -c 'import json, sys; print("\n".join(json.load(sys.stdin)))' \ + | while IFS= read -r nb; do + echo "::group::verify $nb" + docker run --rm \ + -v "$PWD/.github/scripts/run_notebook.py:/ci/run_notebook.py:ro" \ + -v "$RUNNER_TEMP/verify:/out" \ + '${{ steps.prepare.outputs.image }}:candidate' \ + python /ci/run_notebook.py "/work/$nb" --output-dir /out --timeout 3600 + echo "::endgroup::" + done + + - name: Upload verification results + if: always() && steps.check.outputs.up_to_date == 'false' + uses: actions/upload-artifact@v4 + with: + name: verify-${{ matrix.group }} + path: ${{ runner.temp }}/verify/ + retention-days: 30 + if-no-files-found: ignore + + - name: Push verified image + if: steps.check.outputs.up_to_date == 'false' && inputs.push + run: | + IMAGE='${{ steps.prepare.outputs.image }}' + for tag in latest \ + "sha-$(echo '${{ github.sha }}' | cut -c1-7)" \ + "$(date -u +%Y-%m-%d)" \ + 'hash-${{ steps.prepare.outputs.build_hash_short }}'; do + docker tag "$IMAGE:candidate" "$IMAGE:$tag" + docker push "$IMAGE:$tag" + done + + - name: Disk usage telemetry + if: always() + run: df -h diff --git a/README.md b/README.md index df5384b..e1f08a0 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ example-notebooks/ └── / └── / └── / - ├── environment.yml + ├── requirements.in ├── README.md ├── .ipynb ├── .ipynb @@ -22,21 +22,28 @@ For example, [000055/bruntonlab/peterson21](./000055/BruntonLab/peterson21) The `README.md` file should explain the goal of the submission, provide links to relevant scientific publications, and explain the purpose of each notebook file. -The `environment.yml` file should define the dependencies of the environment required for the notebooks to be executed. `environment.yml` files are like `requirements.txt` files, but are designed to work with `conda`. To create this file, follow these steps: +The `requirements.in` file lists the notebooks' **direct** Python dependencies, +one per line — the packages the notebooks actually import (e.g. `dandi`, +`pynwb`, `remfile`, `matplotlib`). Do not list transitive dependencies or +export a full freeze of your environment; our tooling compiles the complete +pinned set from this file. After adding it, run -1. Create a new environment: `conda create -n -python ` -2. Switch into that environment: `conda activate ` -3. Use `conda install ` and `pip install ` to install the necessary dependencies until the notebook(s) run through successfully. -4. Confirm that all the notebooks can be run without error. -5. Export the environment: `conda env export > environment.yml`. - -See [detailed instructions](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#sharing-an-environment) for creating a `environment.yml` file. +```bash +python .github/scripts/lock_notebook.py .ipynb +``` -> **Note:** notebooks are automatically tested in CI and made runnable in Google -> Colab. Before opening a PR, see **[Adding a notebook: CI, Colab, and the -> exclusion lists](docs/adding-notebooks.md)** for the required Colab-bootstrap -> install cell, how the CI test works, headless-execution gotchas, and the -> `.github` exclusion lists. +which resolves the pins against Colab's runtime and writes the install cell +and Colab badge into the notebook for you. If a notebook needs a specific +version range (say it was written against an older matplotlib API), express +that as a bound in `requirements.in` (e.g. `matplotlib<3.11`). + +> **Note:** notebooks are automatically tested in CI, made runnable in Google +> Colab, and published as self-contained [container +> images](.github/docker/README.md). Before opening a PR, see **[Adding a +> notebook: CI, Colab, and the exclusion lists](docs/adding-notebooks.md)** +> for how the CI test works, headless-execution gotchas, and the `.github` +> exclusion lists. (Some older submissions carry an `environment.yml` instead +> of `requirements.in`; new submissions should use `requirements.in`.) Feel free to reach out on the [DANDI helpdesk](https://github.com/dandi/helpdesk/issues/new/choose) with any questions. diff --git a/docs/adding-notebooks.md b/docs/adding-notebooks.md index 4f64c5a..77d8178 100644 --- a/docs/adding-notebooks.md +++ b/docs/adding-notebooks.md @@ -2,18 +2,19 @@ This guide explains what happens to a notebook after you open a Pull Request — how it is tested, how it is made runnable in Google Colab, and the three -`.github/*.txt` lists that control that behavior. For the basic file layout and -`environment.yml`, see the [README](../README.md#submission-instructions); -this doc picks up where that leaves off. +`.github/*.txt` lists that control that behavior. For the basic file layout, +see the [README](../README.md#submission-instructions); this doc picks up +where that leaves off. ## TL;DR — checklist for a new notebook - [ ] Place it under `///` with a `README.md` - and `environment.yml` (see main README). -- [ ] Prepend the **Colab-bootstrap cells** (badge → install intro → pinned - install cell → restart admonition). See [below](#the-colab-bootstrap-cells). -- [ ] Generate the install cell's pins with `uv pip compile`, constrained to - Colab's versions. See [generating the install cell](#generating-the-install-cell). + and a `requirements.in` listing the notebook's **direct** dependencies + (see main README). +- [ ] Run `python .github/scripts/lock_notebook.py .ipynb` to + generate the **Colab-bootstrap cells** (badge → install intro → pinned + install cell → restart admonition). See + [generating the install cell](#generating-the-install-cell). - [ ] **Stream data directly from the DANDI Archive** (remfile/fsspec) — don't download large files or hardcode local paths. See [streaming data](#stream-data-from-the-dandi-archive). @@ -63,7 +64,9 @@ list with a reason. ## The Colab bootstrap cells Every testable notebook begins with four cells (the pattern established in -[PR #149](https://github.com/dandi/example-notebooks/pull/149)): +[PR #149](https://github.com/dandi/example-notebooks/pull/149), generated by +[`lock_notebook.py`](../.github/scripts/lock_notebook.py) — see +[below](#generating-the-install-cell)): 1. **Colab badge** (markdown): ```markdown @@ -98,22 +101,44 @@ set works on Python 3.12 / linux. ## Generating the install cell -Resolve a full, pinned lock with [`uv`](https://github.com/astral-sh/uv), -constrained to **Colab's preinstalled versions** so the install is a no-op for -packages Colab already ships (faster, and avoids the "RESTART RUNTIME" prompt -that changing `numpy` and other C-extensions triggers): +Commit a `requirements.in` next to your notebook listing only its **direct** +dependencies, one per line (e.g. `dandi`, `pynwb`, `remfile`, `matplotlib`, +`pandas`) — not a full freeze, and not transitive packages. Then run: + +```bash +python .github/scripts/lock_notebook.py .ipynb +``` + +The script resolves a full, pinned lock with +[`uv`](https://github.com/astral-sh/uv), constrained to **Colab's preinstalled +versions** so the install is a no-op for packages Colab already ships (faster, +and avoids the "RESTART RUNTIME" prompt that changing `numpy` and other +C-extensions triggers), and writes the four bootstrap cells into the notebook — +prepending them when absent, or refreshing the pin block in place (existing +`!curl`/`!wget` helper lines are preserved). Under the hood it runs: ```bash uv pip compile requirements.in \ --python-version 3.12 \ --python-platform linux \ - --constraint .github/colab-preinstalled.txt \ - -o pins.txt + --constraint .github/colab-preinstalled.txt ``` -where `requirements.in` lists the notebook's **direct** imports (e.g. `dandi`, -`pynwb`, `remfile`, `matplotlib`, `pandas`). Then format each `pkg==ver` line -into the `!uv pip install --system \` block. +Conventions: + +- **One `requirements.in` per directory**, shared by all its notebooks, unless + their needs genuinely differ. Run `lock_notebook.py` on each notebook after + editing it. Notebooks locked from the same file get identical pin sets, so + they are also published together in one + [container image](../.github/docker/README.md). +- A notebook that needs a different dependency set can have its own + `.requirements.in`, which takes precedence over the shared file. +- The `.in` file is **committed**: it is the source of truth for re-locking + when Colab bumps its runtime, a pin breaks, or the periodic image rebuild + needs a fresh resolve. +- If the resolver floats to a version newer than what the notebook was written + against (e.g. a matplotlib release removing a kwarg the notebook uses), add + an upper bound in `requirements.in` (`matplotlib<3.11`) and re-run the script. [`.github/colab-preinstalled.txt`](../.github/colab-preinstalled.txt) is a pip-freeze of the current Colab Python 3.12 runtime (numpy 2.0.2, etc.). Using @@ -122,9 +147,9 @@ are genuinely incompatible with a Colab version, the resolver falls back to a non-Colab version for that package — the user will then get a restart prompt, which is the correct trade-off. -> **nbformat gotcha:** cell `id` fields require `nbformat_minor >= 5`. If you -> prepend cells and validation complains about an unexpected `id`, bump the -> notebook's `nbformat_minor` to 5. +> **nbformat gotcha:** cell `id` fields require `nbformat_minor >= 5`. +> `lock_notebook.py` bumps this automatically when it prepends cells; only +> hand-built bootstrap cells need a manual bump. ## Stream data from the DANDI Archive