From 584f3ee067d956519a325df7d90eaa09025df052 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Sun, 23 Aug 2026 10:26:08 -0400 Subject: [PATCH] Run the harness headlessly for plotly and matplotlib; un-exclude 001170 The ReimerLab notebook was excluded from CI with the note that it calls webbrowser.open. It does not: it builds a plotly express figure and calls fig.show(), and outside a Jupyter kernel plotly resolves its default renderer to "browser", which is what reaches webbrowser.open and raises on a headless runner. Matplotlib has the same shape of problem in the other direction: on a machine with a display it picks a GUI backend, so running the harness locally opens plot windows that CI never sees. Both are properties of the environment rather than of any notebook, so the harness now sets PLOTLY_RENDERER and MPLBACKEND for the ipython path it controls. Colab and JupyterLab inside the notebook image run a real kernel, which selects inline rendering for both by itself, and the kernel path used for PR previews is deliberately left alone so its figures still appear in the executed copy. With that, the notebook runs headlessly, so its exclusion is removed and it gains a container image. It also gets the requirements.in the submission contract expects, which re-locks it onto the 3.13 pin set, and streams through a remfile.DiskCache like the other notebooks that read one file repeatedly. Verified locally on Python 3.13: it now runs in 43 s, where before the cache it took 826 s. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011NuStoQykizhQCx2KBrMCN --- .github/notebook-test-exclusions.txt | 4 - .github/scripts/run_notebook.py | 13 ++- .../ReimerLab/public_demo/001170_demo.ipynb | 103 +++++++++--------- 001170/ReimerLab/public_demo/requirements.in | 10 ++ 4 files changed, 74 insertions(+), 56 deletions(-) create mode 100644 001170/ReimerLab/public_demo/requirements.in diff --git a/.github/notebook-test-exclusions.txt b/.github/notebook-test-exclusions.txt index 454069b5..cc0ac929 100644 --- a/.github/notebook-test-exclusions.txt +++ b/.github/notebook-test-exclusions.txt @@ -61,10 +61,6 @@ tutorials/cosyne_2023/advanced_asset_search.ipynb # Works in Colab (which has libxcb preinstalled). 000559/dattalab/markowitz_gillis_nature_2023/read_avi.ipynb -# Calls webbrowser.open(), which raises "could not locate runnable browser" -# in headless CI. Works in Colab. -001170/ReimerLab/public_demo/001170_demo.ipynb - # ============================================================================= # Pre-existing notebook content bugs — track + fix in dedicated PRs. # ============================================================================= diff --git a/.github/scripts/run_notebook.py b/.github/scripts/run_notebook.py index a4812755..ae329390 100644 --- a/.github/scripts/run_notebook.py +++ b/.github/scripts/run_notebook.py @@ -25,6 +25,7 @@ import argparse import copy import json +import os import re import subprocess import sys @@ -227,8 +228,18 @@ def finalize(stage: str, ok: bool, error: str | None = None, **extra) -> int: log_f = log_path.open("a") log_f.write(f"\n=== execute (streaming) ===\n") log_f.flush() + # Force both plotting libraries into a headless mode for this path. Outside + # a Jupyter kernel plotly resolves its default renderer to "browser", and + # fig.show() then calls webbrowser.open, which raises on a headless runner; + # matplotlib likewise picks a GUI backend when one is available, which opens + # windows when the harness is run on a developer's machine. A real kernel + # (Colab, or JupyterLab in the notebook image) selects inline rendering for + # both by itself, so neither setting applies to the kernel path above. + env = {**os.environ, + "PLOTLY_RENDERER": os.environ.get("PLOTLY_RENDERER", "json"), + "MPLBACKEND": os.environ.get("MPLBACKEND", "Agg")} proc = subprocess.Popen( - cmd, cwd=str(nb_dir), + cmd, cwd=str(nb_dir), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, ) diff --git a/001170/ReimerLab/public_demo/001170_demo.ipynb b/001170/ReimerLab/public_demo/001170_demo.ipynb index c3cd96bd..022e1ced 100644 --- a/001170/ReimerLab/public_demo/001170_demo.ipynb +++ b/001170/ReimerLab/public_demo/001170_demo.ipynb @@ -28,55 +28,56 @@ "outputs": [], "source": [ "#@title Installing requirements (click ▶ to run) { display-mode: \"form\" }\n", - "# Colab provides Python 3.12. We install with `uv --system` because Colab's\n", + "# Colab provides Python 3.13. We install with `uv --system` because Colab's\n", "# kernel runs outside a virtualenv. All versions (direct + transitive) are\n", "# pinned below so the notebook is reproducible regardless of resolver drift.\n", "!pip install -q uv\n", "!uv pip install --system \\\n", " \"acres==0.5.0\" \\\n", - " \"aiohappyeyeballs==2.6.1\" \\\n", - " \"aiohttp==3.13.5\" \\\n", + " \"aiohappyeyeballs==2.7.1\" \\\n", + " \"aiohttp==3.14.3\" \\\n", " \"aiosignal==1.4.0\" \\\n", - " \"annotated-types==0.7.0\" \\\n", + " \"annotated-types==0.8.0\" \\\n", " \"arrow==1.4.0\" \\\n", - " \"asciitree==0.3.3\" \\\n", " \"attrs==26.1.0\" \\\n", - " \"bids-validator-deno==2.4.1\" \\\n", - " \"bidsschematools==1.2.2\" \\\n", - " \"blessed==1.39.0\" \\\n", - " \"certifi==2026.4.22\" \\\n", - " \"cffi==2.0.0\" \\\n", - " \"charset-normalizer==3.4.7\" \\\n", + " \"bids-validator-deno==3.0.1\" \\\n", + " \"bidsschematools==1.2.7\" \\\n", + " \"blessed==1.48.0\" \\\n", + " \"certifi==2026.7.22\" \\\n", + " \"cffi==2.1.1\" \\\n", + " \"charset-normalizer==3.4.9\" \\\n", " \"ci-info==0.4.0\" \\\n", - " \"click==8.1.8\" \\\n", + " \"click==8.4.2\" \\\n", " \"click-didyoumean==0.3.1\" \\\n", " \"contourpy==1.3.3\" \\\n", - " \"cryptography==48.0.0\" \\\n", + " \"cryptography==50.0.0\" \\\n", " \"cycler==0.12.1\" \\\n", - " \"dandi==0.75.1\" \\\n", - " \"dandischema==0.12.1\" \\\n", - " \"deprecated==1.3.1\" \\\n", + " \"dandi==0.77.0\" \\\n", + " \"dandischema==0.14.0\" \\\n", + " \"deno==2.9.5\" \\\n", " \"dnspython==2.8.0\" \\\n", + " \"donfig==0.8.1.post1\" \\\n", " \"email-validator==2.3.0\" \\\n", " \"etelemetry==0.3.1\" \\\n", " \"fasteners==0.20\" \\\n", - " \"fonttools==4.62.1\" \\\n", + " \"fonttools==4.63.0\" \\\n", " \"fqdn==1.5.1\" \\\n", " \"frozenlist==1.8.0\" \\\n", " \"fscacher==0.4.4\" \\\n", - " \"fsspec==2026.4.0\" \\\n", + " \"fsspec==2025.3.0\" \\\n", + " \"google-crc32c==1.8.0\" \\\n", " \"h5py==3.16.0\" \\\n", - " \"hdmf==4.2.0\" \\\n", - " \"hdmf-zarr==0.12.0\" \\\n", - " \"humanize==4.15.0\" \\\n", - " \"idna==3.14\" \\\n", + " \"hdmf==4.3.1\" \\\n", + " \"humanize==4.16.0\" \\\n", + " \"idna==3.18\" \\\n", " \"interleave==0.3.0\" \\\n", " \"isodate==0.7.2\" \\\n", " \"isoduration==20.11.0\" \\\n", " \"jaraco-classes==3.4.0\" \\\n", " \"jaraco-context==6.1.2\" \\\n", - " \"jaraco-functools==4.4.0\" \\\n", + " \"jaraco-functools==4.6.0\" \\\n", " \"jeepney==0.9.0\" \\\n", + " \"jinxed==2.1.0\" \\\n", " \"joblib==1.5.3\" \\\n", " \"jsonpointer==3.1.1\" \\\n", " \"jsonschema==4.26.0\" \\\n", @@ -84,56 +85,54 @@ " \"keyring==25.7.0\" \\\n", " \"keyrings-alt==5.0.2\" \\\n", " \"kiwisolver==1.5.0\" \\\n", - " \"matplotlib==3.10.9\" \\\n", - " \"ml-dtypes==0.5.4\" \\\n", - " \"more-itertools==11.0.2\" \\\n", + " \"matplotlib==3.10.0\" \\\n", + " \"ml-dtypes==0.6.0\" \\\n", + " \"more-itertools==10.8.0\" \\\n", " \"multidict==6.7.1\" \\\n", - " \"narwhals==2.21.0\" \\\n", " \"natsort==8.4.0\" \\\n", - " \"numcodecs==0.15.1\" \\\n", - " \"numpy==2.4.4\" \\\n", - " \"nwbinspector==0.7.1\" \\\n", - " \"packaging==26.2\" \\\n", - " \"pandas==3.0.3\" \\\n", - " \"pillow==12.2.0\" \\\n", - " \"platformdirs==4.9.6\" \\\n", - " \"plotly==6.7.0\" \\\n", + " \"numcodecs==0.16.5\" \\\n", + " \"numpy==2.1.3\" \\\n", + " \"nwbinspector==0.7.2\" \\\n", + " \"packaging==26.3\" \\\n", + " \"pandas==2.2.3\" \\\n", + " \"pillow==11.3.0\" \\\n", + " \"platformdirs==4.11.3\" \\\n", + " \"plotly==5.24.1\" \\\n", " \"propcache==0.5.2\" \\\n", " \"pycparser==3.0\" \\\n", " \"pycryptodomex==3.23.0\" \\\n", " \"pydantic==2.13.4\" \\\n", " \"pydantic-core==2.46.4\" \\\n", - " \"pydantic-settings==2.14.1\" \\\n", + " \"pydantic-settings==2.15.0\" \\\n", " \"pynwb==3.1.3\" \\\n", " \"pyout==0.8.1\" \\\n", " \"pyparsing==3.3.2\" \\\n", " \"python-dateutil==2.9.0.post0\" \\\n", - " \"python-dotenv==1.2.2\" \\\n", + " \"python-dotenv==1.2.3\" \\\n", + " \"pytz==2025.2\" \\\n", " \"pyyaml==6.0.3\" \\\n", " \"referencing==0.37.0\" \\\n", - " \"remfile==0.1.13\" \\\n", - " \"requests==2.34.0\" \\\n", + " \"remfile==0.1.14\" \\\n", + " \"requests==2.32.4\" \\\n", " \"rfc3339-validator==0.1.4\" \\\n", " \"rfc3987==1.3.8\" \\\n", - " \"rpds-py==0.30.0\" \\\n", + " \"rpds-py==2026.6.3\" \\\n", " \"ruamel-yaml==0.19.1\" \\\n", " \"secretstorage==3.5.0\" \\\n", " \"semantic-version==2.10.0\" \\\n", " \"six==1.17.0\" \\\n", " \"tenacity==9.1.4\" \\\n", - " \"tensorstore==0.1.83\" \\\n", - " \"threadpoolctl==3.6.0\" \\\n", + " \"tensorstore==0.1.85\" \\\n", " \"tqdm==4.67.3\" \\\n", - " \"typing-extensions==4.15.0\" \\\n", - " \"typing-inspection==0.4.2\" \\\n", - " \"tzdata==2026.2\" \\\n", + " \"typing-extensions==4.16.0\" \\\n", + " \"typing-inspection==0.4.4\" \\\n", + " \"tzdata==2026.3\" \\\n", " \"uri-template==1.3.0\" \\\n", - " \"urllib3==2.7.0\" \\\n", - " \"wcwidth==0.7.0\" \\\n", + " \"urllib3==2.5.0\" \\\n", + " \"wcwidth==0.8.2\" \\\n", " \"webcolors==25.10.0\" \\\n", - " \"wrapt==2.1.2\" \\\n", - " \"yarl==1.23.0\" \\\n", - " \"zarr==2.18.7\" \\\n", + " \"yarl==1.24.5\" \\\n", + " \"zarr==3.1.5\" \\\n", " \"zarr-checksum==0.4.7\"" ] }, @@ -250,7 +249,9 @@ "import h5py\n", "import remfile\n", "\n", - "file = remfile.File(s3_url)\n", + "# The disk cache keeps blocks that have already been fetched, so repeated\n", + "# reads of the same part of the file do not go back over the network.\n", + "file = remfile.File(s3_url, disk_cache=remfile.DiskCache(\"nwb-cache\"))\n", "h5_file = h5py.File(file, \"r\")" ] }, diff --git a/001170/ReimerLab/public_demo/requirements.in b/001170/ReimerLab/public_demo/requirements.in new file mode 100644 index 00000000..8e23435d --- /dev/null +++ b/001170/ReimerLab/public_demo/requirements.in @@ -0,0 +1,10 @@ +dandi +h5py +matplotlib +numpy +pandas +plotly +pynwb +remfile +pynwb<4 +hdmf<5