From 7dbe5b34f42ba91462c15d933001f83b9561805d Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 19 Aug 2026 19:38:01 -0400 Subject: [PATCH 1/2] Add CI pipeline publishing verified notebook environment images to GHCR Each Colab-ready notebook group (directory + identical pin set) gets a container image with the pinned dependencies preinstalled, the notebooks and helper files baked in, and a JupyterLab entrypoint. The new workflow builds the image, runs every notebook in the group inside it with the unmodified run_notebook.py harness, and pushes to ghcr.io only when all of them pass, so published images are always verified snapshots. workflow_dispatch only for now, defaulting to the 001550/PaganLab pilot. Verified locally end to end: build (380 MB, linux/amd64), both pilot notebooks green inside the image, JupyterLab serves with the pinned kernel resolved. Co-Authored-By: Claude Fable 5 --- .github/docker/Dockerfile | 51 +++++ .github/docker/README.md | 76 +++++++ .github/scripts/build_notebook_image.py | 233 ++++++++++++++++++++ .github/workflows/build-notebook-images.yml | 151 +++++++++++++ 4 files changed, 511 insertions(+) create mode 100644 .github/docker/Dockerfile create mode 100644 .github/docker/README.md create mode 100644 .github/scripts/build_notebook_image.py create mode 100644 .github/workflows/build-notebook-images.yml diff --git a/.github/docker/Dockerfile b/.github/docker/Dockerfile new file mode 100644 index 0000000..fee61bb --- /dev/null +++ b/.github/docker/Dockerfile @@ -0,0 +1,51 @@ +# syntax=docker/dockerfile:1 +# Environment image for DANDI example notebooks. Built by +# .github/workflows/build-notebook-images.yml from a context prepared by +# .github/scripts/build_notebook_image.py (requirements.txt + work/). +# +# Two separate Python environments by design: +# - the kernel env is the system Python (/usr/local): pins go in with +# `uv pip install --system`, matching both the Colab bootstrap cell and +# run_notebook.py, so the CI harness runs unmodified inside the image; +# - JupyterLab lives in an isolated `uv tool` env so its own dependency +# tree can never upgrade or downgrade anything in the pinned kernel env. + +ARG BASE_IMAGE=python:3.12-slim@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a +FROM ${BASE_IMAGE} + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:0.12.5 /uv /uvx /usr/local/bin/ + +ARG JUPYTERLAB_VERSION=4.6.3 +ENV UV_TOOL_BIN_DIR=/opt/uv-bin +RUN uv tool install "jupyterlab==${JUPYTERLAB_VERSION}" +# JUPYTER_PATH makes the pinned kernelspec under /usr/local win over the +# kernelspec that ships inside the JupyterLab tool env. +ENV PATH="/opt/uv-bin:${PATH}" \ + JUPYTER_PATH=/usr/local/share/jupyter + +# ipykernel and nbformat resolve jointly with the pins in one invocation so +# any conflict fails the build instead of silently changing a pinned version. +COPY requirements.txt /tmp/requirements.txt +RUN uv pip install --system --no-cache -r /tmp/requirements.txt ipykernel nbformat \ + && python -m ipykernel install --name python3 --display-name "Python 3 (pinned)" + +WORKDIR /work +COPY work/ /work/ + +ARG DEFAULT_NOTEBOOK +ARG BUILD_HASH +ARG GIT_SHA=unknown +ARG NOTEBOOKS="" +ENV DEFAULT_NOTEBOOK=${DEFAULT_NOTEBOOK} +LABEL org.opencontainers.image.source="https://github.com/dandi/example-notebooks" \ + org.opencontainers.image.revision="${GIT_SHA}" \ + org.dandiarchive.notebooks="${NOTEBOOKS}" \ + org.dandiarchive.build-hash="${BUILD_HASH}" + +EXPOSE 8888 +CMD jupyter-lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root \ + --ServerApp.default_url="/lab/tree/${DEFAULT_NOTEBOOK}" diff --git a/.github/docker/README.md b/.github/docker/README.md new file mode 100644 index 0000000..f9a713e --- /dev/null +++ b/.github/docker/README.md @@ -0,0 +1,76 @@ +# Containerized Notebook Environments + +Each Colab-ready notebook in this repository is also published as a +self-contained container image on the GitHub Container Registry. The image +bundles the notebook, any helper files it fetches, a JupyterLab server, and +the exact pinned dependency set from the notebook's install cell, +preinstalled. Unlike the Colab path, the image does not depend on Colab's +current Python version or on PyPI still serving the pinned packages, so it +remains runnable long after the hosted environments have moved on. + +## Running a Notebook Image + +``` +docker run --rm -p 8888:8888 ghcr.io/dandi/example-notebooks/001550-paganlab:latest +``` + +Then open the `http://127.0.0.1:8888/lab?token=...` URL printed in the +terminal. JupyterLab opens on the notebook with its dependencies already +installed; the install cell at the top is a no-op and can be skipped. The +notebooks stream data from the DANDI Archive, so network access is still +required at run time. + +Images are built for `linux/amd64`, the platform the dependency pins were +resolved for. On Apple Silicon, Docker Desktop runs them under emulation; +pass `--platform linux/amd64` to silence the platform warning. + +Images are named after the notebook directory (lowercased, with `/` replaced +by `-`). When notebooks in the same directory pin different dependency sets, +the notebook name is appended, e.g. +`ghcr.io/dandi/example-notebooks/000971-lernerlab-seiler-2024-optogenetics-example-notebook`. + +## Tags + +| Tag | Meaning | +| --- | --- | +| `latest` | most recent verified build | +| `YYYY-MM-DD` | date-stamped snapshot, useful as a citable reference | +| `sha-` | git commit the image was built from | +| `hash-<12 hex>` | content hash of the build inputs (pins, notebooks, helpers, Dockerfile); used by CI to skip unchanged rebuilds | + +## How Images Are Built and Verified + +The `Build notebook images` workflow +(`.github/workflows/build-notebook-images.yml`) groups notebooks by directory +and pin set (`.github/scripts/build_notebook_image.py`), builds one image per +group from the parameterized `Dockerfile` in this directory, and then runs +every notebook in the group **inside the candidate image** using the same +harness the test workflows use (`.github/scripts/run_notebook.py`). An image +is pushed only if all of its notebooks execute successfully, so `latest` is +always a verified snapshot. + +Inside the image the kernel environment is the system Python, with pins +installed via `uv pip install --system`, exactly matching the Colab bootstrap +cell and the CI harness. JupyterLab runs from an isolated `uv tool` +environment so its own dependencies cannot perturb the pinned kernel +environment. `ipykernel` and `nbformat` are the only additions to the pinned +set; they resolve jointly with the pins so a conflict fails the build rather +than silently changing a pinned version. + +## Maintainer Notes + +- The workflow is currently `workflow_dispatch` only. The `filter` input is a + regex on the notebook directory or path (default: the `001550/PaganLab` + pilot); `push: false` gives a dry run (build + verify, no publish); `force` + rebuilds even when the build-hash check says the published image is current. +- Rebuilds are skipped when a `hash-<...>` tag matching the current build + inputs already exists on the registry. Changing a notebook, its pins, its + helpers, or the Dockerfile changes the hash and triggers a rebuild. +- The base image is digest-pinned in the Dockerfile (`ARG BASE_IMAGE`). To + bump it, update the digest, which changes every group's build hash, and + dispatch a full rebuild. The `BASE_IMAGE` arg is also the knob for a future + variant based on the official Colab runtime image. +- Images run as root (`python:3.12-slim` has no unprivileged user) and keep + Jupyter's token auth enabled. The published port binding in the docs is + loopback-only via `-p 8888:8888` on a local machine; advise users not to + bind on public interfaces. diff --git a/.github/scripts/build_notebook_image.py b/.github/scripts/build_notebook_image.py new file mode 100644 index 0000000..42b6441 --- /dev/null +++ b/.github/scripts/build_notebook_image.py @@ -0,0 +1,233 @@ +"""Group notebooks into container-image build units and prepare build contexts. + +Notebooks that carry the Colab-bootstrap install cell are grouped by +(directory, pin-set): notebooks in the same directory whose install cells pin +the identical dependency set share one image. The image for a group contains +every file of that directory (minus notebooks belonging to other groups) plus +any helper files the install cells fetch via `!curl`/`!wget`, with the pinned +dependencies preinstalled into the system Python. + +Subcommands: + list-groups [--filter REGEX] [--names-only] + Print the groups as JSON. `--names-only` emits just the group names, + suitable for a GitHub Actions matrix. + prepare --name NAME --context-dir DIR [--image-prefix PREFIX] + Write a docker build context (requirements.txt + work/) for one group + and emit image name, build hash, default notebook, and the notebook + list to $GITHUB_OUTPUT (or stdout when unset). + +The pin extraction reuses `find_install_cell` from run_notebook.py, so the +image contents stay in lockstep with what CI tests. Entries captured by that +regex that are not actual pins (it also picks up the literal `form` from the +`{ display-mode: "form" }` title line) are filtered out: only `pkg==ver` and +`pkg @ url` requirements are baked into an image. + +Assumes `nbformat` is importable. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +import nbformat + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from list_notebooks import REPO_ROOT, is_excluded, load_exclusions # noqa: E402 +from run_notebook import find_install_cell # noqa: E402 + +DOCKERFILE = REPO_ROOT / ".github" / "docker" / "Dockerfile" +DEFAULT_IMAGE_PREFIX = "ghcr.io/dandi/example-notebooks" + + +def slug(s: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") + + +def real_pins(pins: list[str]) -> tuple[list[str], list[str]]: + """Split install-cell entries into actual requirements and strays.""" + kept = [p for p in pins if "==" in p or " @ " in p] + dropped = [p for p in pins if p not in kept] + return kept, dropped + + +@dataclass +class Group: + directory: str # repo-relative, e.g. "001550/PaganLab" + pin_hash: str + pins: list[str] + helpers: list[str] = field(default_factory=list) + notebooks: list[str] = field(default_factory=list) # filenames within directory + name: str = "" + + def as_dict(self) -> dict: + return { + "name": self.name, + "directory": self.directory, + "pin_hash": self.pin_hash, + "notebooks": sorted(self.notebooks), + } + + +def collect_groups() -> list[Group]: + exclusions = load_exclusions() + groups: dict[tuple[str, str], Group] = {} + for path in sorted(REPO_ROOT.rglob("*.ipynb")): + if ".ipynb_checkpoints" in path.parts: + continue + rel = str(path.relative_to(REPO_ROOT)) + if is_excluded(rel, exclusions): + continue + nb = nbformat.read(path, as_version=4) + try: + pins, helpers, _ = find_install_cell(nb) + except RuntimeError: + continue # no Colab bootstrap -> no image + kept, _ = real_pins(pins) + pin_hash = hashlib.sha256("\n".join(sorted(kept)).encode()).hexdigest() + directory = str(path.parent.relative_to(REPO_ROOT)) + key = (directory, pin_hash) + group = groups.setdefault( + key, Group(directory=directory, pin_hash=pin_hash, pins=sorted(kept)) + ) + group.notebooks.append(path.name) + for h in helpers: + if h not in group.helpers: + group.helpers.append(h) + + per_dir: dict[str, list[Group]] = {} + for g in groups.values(): + per_dir.setdefault(g.directory, []).append(g) + for dir_groups in per_dir.values(): + for g in dir_groups: + g.name = slug(g.directory) + if len(dir_groups) > 1: + g.name += "-" + slug(Path(sorted(g.notebooks)[0]).stem) + + result = sorted(groups.values(), key=lambda g: g.name) + names = [g.name for g in result] + assert len(names) == len(set(names)), f"group name collision: {names}" + return result + + +def matches(group: Group, pattern: str) -> bool: + if not pattern: + return True + rx = re.compile(pattern) + return bool( + rx.search(group.directory) + or any(rx.search(f"{group.directory}/{n}") for n in group.notebooks) + ) + + +def cmd_list_groups(args: argparse.Namespace) -> int: + groups = [g for g in collect_groups() if matches(g, args.filter)] + if args.names_only: + print(json.dumps([g.name for g in groups])) + else: + print(json.dumps([g.as_dict() for g in groups], indent=2)) + return 0 + + +def cmd_prepare(args: argparse.Namespace) -> int: + groups = [g for g in collect_groups() if g.name == args.name] + if not groups: + print(f"error: no group named {args.name!r}", file=sys.stderr) + return 1 + group = groups[0] + + context = Path(args.context_dir) + if context.exists(): + shutil.rmtree(context) + work = context / "work" + + group_notebooks = set(group.notebooks) + src_dir = REPO_ROOT / group.directory + + def ignore(directory: str, names: list[str]) -> list[str]: + ignored = [n for n in names if n == ".ipynb_checkpoints"] + for n in names: + if not n.endswith(".ipynb"): + continue + in_group = ( + Path(directory) == src_dir and n in group_notebooks + ) + if not in_group: + ignored.append(n) + return ignored + + shutil.copytree(src_dir, work, ignore=ignore) + + (context / "requirements.txt").write_text("\n".join(group.pins) + "\n") + + # Bake helper files the same way run_notebook.py fetches them in CI. + for h in group.helpers: + cmd_str = h.lstrip("!").strip() + m = re.search(r"-o\s+(\S+)", cmd_str) + if m: + target = work / m.group(1) + target.parent.mkdir(parents=True, exist_ok=True) + r = subprocess.run( + ["bash", "-c", cmd_str], cwd=str(work), capture_output=True, text=True + ) + if r.returncode != 0: + print(f"error: helper failed: {cmd_str}\n{r.stderr[-1000:]}", file=sys.stderr) + return 1 + + h = hashlib.sha256() + for pin in group.pins: + h.update(pin.encode() + b"\n") + for helper in group.helpers: + h.update(helper.encode() + b"\n") + for name in sorted(group.notebooks): + h.update((REPO_ROOT / group.directory / name).read_bytes()) + h.update(DOCKERFILE.read_bytes()) + build_hash = h.hexdigest() + + notebooks = sorted(group.notebooks) + outputs = { + "image": f"{args.image_prefix}/{group.name}", + "build_hash": build_hash, + "build_hash_short": build_hash[:12], + "default_notebook": notebooks[0], + "notebooks": json.dumps(notebooks), + "context_dir": str(context), + } + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with open(github_output, "a") as f: + for k, v in outputs.items(): + f.write(f"{k}={v}\n") + print(json.dumps(outputs, indent=2)) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + p_list = sub.add_parser("list-groups") + p_list.add_argument("--filter", default="", help="regex on directory or notebook path") + p_list.add_argument("--names-only", action="store_true") + p_list.set_defaults(func=cmd_list_groups) + + p_prep = sub.add_parser("prepare") + p_prep.add_argument("--name", required=True) + p_prep.add_argument("--context-dir", required=True) + p_prep.add_argument("--image-prefix", default=DEFAULT_IMAGE_PREFIX) + p_prep.set_defaults(func=cmd_prepare) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/build-notebook-images.yml b/.github/workflows/build-notebook-images.yml new file mode 100644 index 0000000..08ef2e9 --- /dev/null +++ b/.github/workflows/build-notebook-images.yml @@ -0,0 +1,151 @@ +name: Build notebook images + +# Builds a container image per notebook group (directory + identical pin set), +# verifies every notebook in the group runs inside the candidate image using +# the same harness as the test workflows, and pushes verified images to +# ghcr.io. Only verified images are ever published. + +on: + workflow_dispatch: + inputs: + filter: + description: "Regex; build only groups whose directory or notebook path matches" + default: "^001550/PaganLab" + force: + description: "Rebuild and push even if an image with this build hash exists" + type: boolean + default: false + push: + description: "Push to ghcr.io (uncheck for a dry-run build + verify)" + type: boolean + default: true + +permissions: + contents: read + packages: write + +jobs: + list: + runs-on: ubuntu-latest + outputs: + groups: ${{ steps.groups.outputs.groups }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install runner dependencies + run: pip install --no-cache-dir nbformat + + - name: List image groups + id: groups + run: | + python .github/scripts/build_notebook_image.py list-groups \ + --filter '${{ github.event.inputs.filter }}' --names-only > groups.json + echo "groups=$(cat groups.json)" >> "$GITHUB_OUTPUT" + echo "Matched groups:" + cat groups.json + + build: + needs: list + if: needs.list.outputs.groups != '[]' + runs-on: ubuntu-latest + timeout-minutes: 90 + strategy: + fail-fast: false + max-parallel: 6 + matrix: + group: ${{ fromJSON(needs.list.outputs.groups) }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install runner dependencies + run: pip install --no-cache-dir nbformat + + - name: Prepare build context + id: prepare + run: | + python .github/scripts/build_notebook_image.py prepare \ + --name '${{ matrix.group }}' \ + --context-dir "$RUNNER_TEMP/context" \ + --image-prefix "ghcr.io/${{ github.repository }}" + + - name: Log in to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Skip if an image with this build hash is already published + id: check + run: | + IMAGE='${{ steps.prepare.outputs.image }}' + TAG='hash-${{ steps.prepare.outputs.build_hash_short }}' + if [ '${{ inputs.force }}' != 'true' ] \ + && docker manifest inspect "$IMAGE:$TAG" >/dev/null 2>&1; then + echo "Image $IMAGE:$TAG already exists; skipping build and push." + echo "up_to_date=true" >> "$GITHUB_OUTPUT" + else + echo "up_to_date=false" >> "$GITHUB_OUTPUT" + fi + + - name: Build candidate image + if: steps.check.outputs.up_to_date == 'false' + run: | + docker build \ + --platform linux/amd64 \ + -f .github/docker/Dockerfile \ + --build-arg DEFAULT_NOTEBOOK='${{ steps.prepare.outputs.default_notebook }}' \ + --build-arg BUILD_HASH='${{ steps.prepare.outputs.build_hash }}' \ + --build-arg GIT_SHA='${{ github.sha }}' \ + --build-arg NOTEBOOKS='${{ steps.prepare.outputs.notebooks }}' \ + -t '${{ steps.prepare.outputs.image }}:candidate' \ + "$RUNNER_TEMP/context" + + - name: Verify notebooks inside the candidate image + if: steps.check.outputs.up_to_date == 'false' + run: | + mkdir -p "$RUNNER_TEMP/verify" + echo '${{ steps.prepare.outputs.notebooks }}' \ + | python3 -c 'import json, sys; print("\n".join(json.load(sys.stdin)))' \ + | while IFS= read -r nb; do + echo "::group::verify $nb" + docker run --rm \ + -v "$PWD/.github/scripts/run_notebook.py:/ci/run_notebook.py:ro" \ + -v "$RUNNER_TEMP/verify:/out" \ + '${{ steps.prepare.outputs.image }}:candidate' \ + python /ci/run_notebook.py "/work/$nb" --output-dir /out --timeout 3600 + echo "::endgroup::" + done + + - name: Upload verification results + if: always() && steps.check.outputs.up_to_date == 'false' + uses: actions/upload-artifact@v4 + with: + name: verify-${{ matrix.group }} + path: ${{ runner.temp }}/verify/ + retention-days: 30 + if-no-files-found: ignore + + - name: Push verified image + if: steps.check.outputs.up_to_date == 'false' && inputs.push + run: | + IMAGE='${{ steps.prepare.outputs.image }}' + for tag in latest \ + "sha-$(echo '${{ github.sha }}' | cut -c1-7)" \ + "$(date -u +%Y-%m-%d)" \ + 'hash-${{ steps.prepare.outputs.build_hash_short }}'; do + docker tag "$IMAGE:candidate" "$IMAGE:$tag" + docker push "$IMAGE:$tag" + done + + - name: Disk usage telemetry + if: always() + run: df -h From 3b099a9a830b70eccb22340700bafe5abb81d3f3 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 19 Aug 2026 19:51:46 -0400 Subject: [PATCH 2/2] Make requirements.in the submission contract; add lock_notebook.py Contributors now commit a requirements.in listing only their notebook's direct dependencies, and lock_notebook.py assembles the rest: it compiles the full pinned set with uv against Colab's preinstalled versions and writes the four bootstrap cells into the notebook, prepending them when absent or refreshing the pin block in place (helper fetch lines are preserved, and nbformat_minor is bumped when cells are prepended). Committing the .in file also gives re-locking a source of truth: today the pins only exist inside the notebook JSON, so re-resolving after a Colab runtime bump means reverse-engineering the direct deps from a 160-line pin list. The container image pipeline rebuilds flow from the same file. README submission instructions now ask for requirements.in instead of a conda env export, docs/adding-notebooks.md documents the script and the one-file-per-directory convention, and the CI missing-install-cell error points at the new flow. Co-Authored-By: Claude Fable 5 --- .github/docker/README.md | 4 +- .github/scripts/lock_notebook.py | 158 +++++++++++++++++++++++++++++++ .github/scripts/run_notebook.py | 4 +- README.md | 35 ++++--- docs/adding-notebooks.md | 67 +++++++++---- 5 files changed, 231 insertions(+), 37 deletions(-) create mode 100644 .github/scripts/lock_notebook.py diff --git a/.github/docker/README.md b/.github/docker/README.md index f9a713e..5544cbc 100644 --- a/.github/docker/README.md +++ b/.github/docker/README.md @@ -65,7 +65,9 @@ than silently changing a pinned version. rebuilds even when the build-hash check says the published image is current. - Rebuilds are skipped when a `hash-<...>` tag matching the current build inputs already exists on the registry. Changing a notebook, its pins, its - helpers, or the Dockerfile changes the hash and triggers a rebuild. + helpers, or the Dockerfile changes the hash and triggers a rebuild. Pins are + refreshed with `.github/scripts/lock_notebook.py` from the `requirements.in` + committed next to the notebook (see `docs/adding-notebooks.md`). - The base image is digest-pinned in the Dockerfile (`ARG BASE_IMAGE`). To bump it, update the digest, which changes every group's build hash, and dispatch a full rebuild. The `BASE_IMAGE` arg is also the knob for a future diff --git a/.github/scripts/lock_notebook.py b/.github/scripts/lock_notebook.py new file mode 100644 index 0000000..6531a9a --- /dev/null +++ b/.github/scripts/lock_notebook.py @@ -0,0 +1,158 @@ +"""Generate or refresh a notebook's Colab-bootstrap cells from a requirements.in. + +Contributors commit a `requirements.in` next to their notebook listing only the +notebook's direct dependencies (e.g. `pynwb`, `remfile`, `matplotlib`). This +script compiles that into a fully pinned set with `uv pip compile`, constrained +to Colab's preinstalled versions, and writes the four bootstrap cells (badge, +install intro, pinned install cell, restart admonition) into the notebook — +prepending them when absent, or refreshing the install cell's pin block in +place (helper `!curl`/`!wget` lines are preserved) when already present. + +The requirements file is resolved per notebook: `.requirements.in` next +to the notebook wins when present, otherwise the directory's `requirements.in` +is used. One `requirements.in` shared by all notebooks in a directory keeps +their pin sets identical, so they are tested against one environment and are +published together in one container image (see ../docker/README.md). + +Usage: + python .github/scripts/lock_notebook.py [...] + +Assumes `uv` is on PATH and `nbformat` is importable. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +import nbformat + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from list_notebooks import REPO_ROOT # noqa: E402 +from run_notebook import find_install_cell # noqa: E402 + +CONSTRAINT = REPO_ROOT / ".github" / "colab-preinstalled.txt" + +BADGE = ( + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)]" + "(https://colab.research.google.com/github/dandi/example-notebooks/blob/master/{path})" +) +INTRO = ( + "## Installing requirements\n" + "\n" + "The cell below installs every Python package needed to run this notebook, " + "at fully pinned versions, using [`uv`](https://github.com/astral-sh/uv) for " + "fast resolution. In Colab the cell is collapsed by default — click the " + "▶ button to run it." +) +RESTART = ( + "> **⚠️ Restart runtime after install**\n" + ">\n" + "> The install may upgrade packages already loaded in the kernel. Go to " + "**Runtime → Restart session**, then **Run all cells below** (skip this " + "install cell on re-run)." +) +INSTALL_HEADER = ( + '#@title Installing requirements (click ▶ to run) { display-mode: "form" }\n' + "# Colab provides Python 3.12. We install with `uv --system` because Colab's\n" + "# kernel runs outside a virtualenv. All versions (direct + transitive) are\n" + "# pinned below so the notebook is reproducible regardless of resolver drift.\n" + "!pip install -q uv\n" +) + + +def requirements_for(nb_path: Path) -> Path: + per_notebook = nb_path.with_name(f"{nb_path.stem}.requirements.in") + if per_notebook.exists(): + return per_notebook + shared = nb_path.parent / "requirements.in" + if shared.exists(): + return shared + raise FileNotFoundError( + f"No requirements file for {nb_path}: expected {per_notebook.name} or " + f"requirements.in in {nb_path.parent}/. List the notebook's direct " + "dependencies there (one per line), then re-run this script." + ) + + +def compile_pins(requirements: Path) -> list[str]: + cmd = [ + "uv", "pip", "compile", str(requirements), + "--python-version", "3.12", + "--python-platform", "linux", + "--constraint", str(CONSTRAINT), + "--no-header", "--no-annotate", + ] + r = subprocess.run(cmd, capture_output=True, text=True, stdin=subprocess.DEVNULL) + if r.returncode != 0: + raise RuntimeError(f"uv pip compile failed for {requirements}:\n{r.stderr}") + pins = [ + line.strip() + for line in r.stdout.splitlines() + if line.strip() and not line.strip().startswith("#") + ] + if not pins: + raise RuntimeError(f"uv pip compile produced no pins from {requirements}") + return pins + + +def install_cell_source(pins: list[str], helpers: list[str]) -> str: + lines = [INSTALL_HEADER + "!uv pip install --system \\"] + lines += [f' "{pin}" \\' for pin in pins[:-1]] + lines.append(f' "{pins[-1]}"') + lines += helpers + return "\n".join(lines) + + +def lock(nb_path: Path) -> None: + requirements = requirements_for(nb_path) + pins = compile_pins(requirements) + nb = nbformat.read(nb_path, as_version=4) + + try: + _, helpers, install_idx = find_install_cell(nb) + except RuntimeError: + helpers, install_idx = [], None + + if install_idx is not None: + nb.cells[install_idx].source = install_cell_source(pins, helpers) + nb.cells[install_idx].metadata["cellView"] = "form" + action = "refreshed install cell in" + else: + rel = nb_path.resolve().relative_to(REPO_ROOT) + install_cell = nbformat.v4.new_code_cell(install_cell_source(pins, helpers)) + install_cell.metadata["cellView"] = "form" + nb.cells = [ + nbformat.v4.new_markdown_cell(BADGE.format(path=str(rel).replace(" ", "%20"))), + nbformat.v4.new_markdown_cell(INTRO), + install_cell, + nbformat.v4.new_markdown_cell(RESTART), + ] + nb.cells + # Cell ids require nbformat 4.5; prepended cells carry ids. + nb.nbformat_minor = max(nb.nbformat_minor, 5) + action = "prepended bootstrap cells to" + + nbformat.validate(nb) + nbformat.write(nb, nb_path) + print(f"{action} {nb_path} ({len(pins)} pins from {requirements.name}" + + (f", kept {len(helpers)} helper line(s)" if helpers else "") + ")") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("notebooks", nargs="+", type=Path) + args = parser.parse_args() + failures = 0 + for nb_path in args.notebooks: + try: + lock(nb_path) + except Exception as e: + print(f"error: {nb_path}: {e}", file=sys.stderr) + failures += 1 + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/run_notebook.py b/.github/scripts/run_notebook.py index 27bd88c..94b66ee 100644 --- a/.github/scripts/run_notebook.py +++ b/.github/scripts/run_notebook.py @@ -56,7 +56,9 @@ def find_install_cell(nb): "CI walks every notebook in the repo and expects each one to begin with a " "code cell containing `!uv pip install --system \"pkg==ver\" ...` so that " "its dep set is fully pinned and reproducible. " - "To fix: add the bootstrap cells (see PR #149 for the pattern) — OR, if " + "To fix: commit a `requirements.in` next to the notebook and run " + "`python .github/scripts/lock_notebook.py ` to generate the " + "bootstrap cells (see docs/adding-notebooks.md) — OR, if " "the notebook genuinely can't be tested headlessly (needs a database, " "proprietary creds, etc), add its path to `.github/notebook-test-exclusions.txt` " "with a comment explaining why." diff --git a/README.md b/README.md index df5384b..e1f08a0 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ example-notebooks/ └── / └── / └── / - ├── environment.yml + ├── requirements.in ├── README.md ├── .ipynb ├── .ipynb @@ -22,21 +22,28 @@ For example, [000055/bruntonlab/peterson21](./000055/BruntonLab/peterson21) The `README.md` file should explain the goal of the submission, provide links to relevant scientific publications, and explain the purpose of each notebook file. -The `environment.yml` file should define the dependencies of the environment required for the notebooks to be executed. `environment.yml` files are like `requirements.txt` files, but are designed to work with `conda`. To create this file, follow these steps: +The `requirements.in` file lists the notebooks' **direct** Python dependencies, +one per line — the packages the notebooks actually import (e.g. `dandi`, +`pynwb`, `remfile`, `matplotlib`). Do not list transitive dependencies or +export a full freeze of your environment; our tooling compiles the complete +pinned set from this file. After adding it, run -1. Create a new environment: `conda create -n -python ` -2. Switch into that environment: `conda activate ` -3. Use `conda install ` and `pip install ` to install the necessary dependencies until the notebook(s) run through successfully. -4. Confirm that all the notebooks can be run without error. -5. Export the environment: `conda env export > environment.yml`. - -See [detailed instructions](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#sharing-an-environment) for creating a `environment.yml` file. +```bash +python .github/scripts/lock_notebook.py .ipynb +``` -> **Note:** notebooks are automatically tested in CI and made runnable in Google -> Colab. Before opening a PR, see **[Adding a notebook: CI, Colab, and the -> exclusion lists](docs/adding-notebooks.md)** for the required Colab-bootstrap -> install cell, how the CI test works, headless-execution gotchas, and the -> `.github` exclusion lists. +which resolves the pins against Colab's runtime and writes the install cell +and Colab badge into the notebook for you. If a notebook needs a specific +version range (say it was written against an older matplotlib API), express +that as a bound in `requirements.in` (e.g. `matplotlib<3.11`). + +> **Note:** notebooks are automatically tested in CI, made runnable in Google +> Colab, and published as self-contained [container +> images](.github/docker/README.md). Before opening a PR, see **[Adding a +> notebook: CI, Colab, and the exclusion lists](docs/adding-notebooks.md)** +> for how the CI test works, headless-execution gotchas, and the `.github` +> exclusion lists. (Some older submissions carry an `environment.yml` instead +> of `requirements.in`; new submissions should use `requirements.in`.) Feel free to reach out on the [DANDI helpdesk](https://github.com/dandi/helpdesk/issues/new/choose) with any questions. diff --git a/docs/adding-notebooks.md b/docs/adding-notebooks.md index 4f64c5a..77d8178 100644 --- a/docs/adding-notebooks.md +++ b/docs/adding-notebooks.md @@ -2,18 +2,19 @@ This guide explains what happens to a notebook after you open a Pull Request — how it is tested, how it is made runnable in Google Colab, and the three -`.github/*.txt` lists that control that behavior. For the basic file layout and -`environment.yml`, see the [README](../README.md#submission-instructions); -this doc picks up where that leaves off. +`.github/*.txt` lists that control that behavior. For the basic file layout, +see the [README](../README.md#submission-instructions); this doc picks up +where that leaves off. ## TL;DR — checklist for a new notebook - [ ] Place it under `///` with a `README.md` - and `environment.yml` (see main README). -- [ ] Prepend the **Colab-bootstrap cells** (badge → install intro → pinned - install cell → restart admonition). See [below](#the-colab-bootstrap-cells). -- [ ] Generate the install cell's pins with `uv pip compile`, constrained to - Colab's versions. See [generating the install cell](#generating-the-install-cell). + and a `requirements.in` listing the notebook's **direct** dependencies + (see main README). +- [ ] Run `python .github/scripts/lock_notebook.py .ipynb` to + generate the **Colab-bootstrap cells** (badge → install intro → pinned + install cell → restart admonition). See + [generating the install cell](#generating-the-install-cell). - [ ] **Stream data directly from the DANDI Archive** (remfile/fsspec) — don't download large files or hardcode local paths. See [streaming data](#stream-data-from-the-dandi-archive). @@ -63,7 +64,9 @@ list with a reason. ## The Colab bootstrap cells Every testable notebook begins with four cells (the pattern established in -[PR #149](https://github.com/dandi/example-notebooks/pull/149)): +[PR #149](https://github.com/dandi/example-notebooks/pull/149), generated by +[`lock_notebook.py`](../.github/scripts/lock_notebook.py) — see +[below](#generating-the-install-cell)): 1. **Colab badge** (markdown): ```markdown @@ -98,22 +101,44 @@ set works on Python 3.12 / linux. ## Generating the install cell -Resolve a full, pinned lock with [`uv`](https://github.com/astral-sh/uv), -constrained to **Colab's preinstalled versions** so the install is a no-op for -packages Colab already ships (faster, and avoids the "RESTART RUNTIME" prompt -that changing `numpy` and other C-extensions triggers): +Commit a `requirements.in` next to your notebook listing only its **direct** +dependencies, one per line (e.g. `dandi`, `pynwb`, `remfile`, `matplotlib`, +`pandas`) — not a full freeze, and not transitive packages. Then run: + +```bash +python .github/scripts/lock_notebook.py .ipynb +``` + +The script resolves a full, pinned lock with +[`uv`](https://github.com/astral-sh/uv), constrained to **Colab's preinstalled +versions** so the install is a no-op for packages Colab already ships (faster, +and avoids the "RESTART RUNTIME" prompt that changing `numpy` and other +C-extensions triggers), and writes the four bootstrap cells into the notebook — +prepending them when absent, or refreshing the pin block in place (existing +`!curl`/`!wget` helper lines are preserved). Under the hood it runs: ```bash uv pip compile requirements.in \ --python-version 3.12 \ --python-platform linux \ - --constraint .github/colab-preinstalled.txt \ - -o pins.txt + --constraint .github/colab-preinstalled.txt ``` -where `requirements.in` lists the notebook's **direct** imports (e.g. `dandi`, -`pynwb`, `remfile`, `matplotlib`, `pandas`). Then format each `pkg==ver` line -into the `!uv pip install --system \` block. +Conventions: + +- **One `requirements.in` per directory**, shared by all its notebooks, unless + their needs genuinely differ. Run `lock_notebook.py` on each notebook after + editing it. Notebooks locked from the same file get identical pin sets, so + they are also published together in one + [container image](../.github/docker/README.md). +- A notebook that needs a different dependency set can have its own + `.requirements.in`, which takes precedence over the shared file. +- The `.in` file is **committed**: it is the source of truth for re-locking + when Colab bumps its runtime, a pin breaks, or the periodic image rebuild + needs a fresh resolve. +- If the resolver floats to a version newer than what the notebook was written + against (e.g. a matplotlib release removing a kwarg the notebook uses), add + an upper bound in `requirements.in` (`matplotlib<3.11`) and re-run the script. [`.github/colab-preinstalled.txt`](../.github/colab-preinstalled.txt) is a pip-freeze of the current Colab Python 3.12 runtime (numpy 2.0.2, etc.). Using @@ -122,9 +147,9 @@ are genuinely incompatible with a Colab version, the resolver falls back to a non-Colab version for that package — the user will then get a restart prompt, which is the correct trade-off. -> **nbformat gotcha:** cell `id` fields require `nbformat_minor >= 5`. If you -> prepend cells and validation complains about an unexpected `id`, bump the -> notebook's `nbformat_minor` to 5. +> **nbformat gotcha:** cell `id` fields require `nbformat_minor >= 5`. +> `lock_notebook.py` bumps this automatically when it prepends cells; only +> hand-built bootstrap cells need a manual bump. ## Stream data from the DANDI Archive