From 67f3427acc439bd4a0d379a65532d1b09212eb0d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 05:16:44 +0000 Subject: [PATCH 1/3] Avoid rebuilding pyo3 on every CI run by pinning PYO3_CONFIG_FILE PEP 517 builds run in a randomly-named temporary environment, and PyO3's build script fingerprints the interpreter path it is given by maturin. That path changes on every CI run, which makes cargo recompile pyo3-build-config, pyo3-ffi, pyo3, pyo3-macros-backend, and pyo3-macros (and on Windows openssl-sys/openssl too) despite a warm rust-cache. Instead of disabling build isolation, resolve the PyO3 build config once per session -- against the session venv's interpreter, whose path is stable from run to run -- using pyo3's own discovery (PYO3_PRINT_CONFIG), and pin it via PYO3_CONFIG_FILE. PYO3_CONFIG_FILE takes priority over the PYO3_PYTHON that maturin points at the ephemeral build environment, so cached pyo3 artifacts stay fresh while the build itself remains fully isolated. The config file's name is content-addressed and its mtime pinned so that a regenerated-but-identical config doesn't trip cargo's rerun-if-changed, while a genuinely different config changes the PYO3_CONFIG_FILE value and correctly triggers a rebuild. The cache key is bumped so caches are re-saved with the new fingerprints (rust-cache does not re-save on a primary key hit). https://claude.ai/code/session_014StKTjk7GBcVdiWKimEsQb --- .github/actions/cache/action.yml | 2 +- noxfile.py | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index 8e729e66e4a9..ce61c3c2d708 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -19,4 +19,4 @@ runs: KEY: "${{ inputs.key }}" - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - key: ${{ steps.normalized-key.outputs.key }}-5 + key: ${{ steps.normalized-key.outputs.key }}-7 diff --git a/noxfile.py b/noxfile.py index aa34f95b406b..ca7e4b6778ff 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,6 +5,7 @@ from __future__ import annotations import glob +import hashlib import itertools import json import os @@ -44,6 +45,56 @@ def load_pyproject_toml() -> dict: return tomllib.load(f) +def pin_pyo3_config(session: nox.Session) -> None: + # PEP 517 builds run in a randomly-named temporary environment, and + # PyO3's build script fingerprints the interpreter path it is given. + # That makes cargo recompile pyo3 (and everything downstream of it) on + # every CI run, despite an otherwise warm cache. Resolve the PyO3 + # build config once, against this session's interpreter (whose path is + # stable from run to run), and pin it via PYO3_CONFIG_FILE so the + # build is independent of the ephemeral build environment. + venv = pathlib.Path(session.virtualenv.location) + python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + output = session.run_install( + "cargo", + "check", + "-p", + "pyo3-ffi", + external=True, + silent=True, + success_codes=[0, 101], + env={"PYO3_PRINT_CONFIG": "1", "PYO3_PYTHON": str(python)}, + ) + # None on --no-install invocations, where the config file (and the + # build it is for) already exists. + if output is None: + return + assert isinstance(output, str) + config_lines = [] + in_config = False + for line in output.splitlines(): + line = line.strip() + if "PYO3_PRINT_CONFIG=1 is set" in line: + in_config = True + continue + if in_config: + if line.startswith("note:") or "=" not in line: + break + config_lines.append(line) + assert config_lines, f"failed to extract PyO3 config from:\n{output}" + content = "\n".join(config_lines) + "\n" + # The file name is content-addressed and the mtime is pinned: cargo + # treats both a changed PYO3_CONFIG_FILE value and a fresh mtime on + # the file it points to as reasons to recompile pyo3. This way a + # regenerated-but-identical config (e.g. a fresh CI run) never looks + # changed, while a genuinely different config gets a new path. + digest = hashlib.sha256(content.encode()).hexdigest()[:16] + config_path = venv / f"pyo3-config-{digest}.txt" + config_path.write_text(content) + os.utime(config_path, (0, 0)) + session.env["PYO3_CONFIG_FILE"] = str(config_path) + + @nox.session @nox.session(name="tests-ssh") @nox.session(name="tests-randomorder") @@ -71,6 +122,7 @@ def tests(session: nox.Session) -> None: ) install_spec = f".[{','.join(extras)}]" + pin_pyo3_config(session) install(session, "-e", "./vectors") if session.name == "tests-rust-debug": install( From 74db9cd51ef041faa3c662a861e3494476c2c782 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 05:23:46 +0000 Subject: [PATCH 2/3] Include ext_suffix and abiflags in the pinned PyO3 config for maturin maturin also reads PYO3_CONFIG_FILE and treats it as the authoritative description of the target interpreter: it requires ext_suffix on Windows, and without abiflags it tags free-threaded wheels as regular CPython (cp314 instead of cp314t), which the installer then rejects. PyO3 itself ignores unknown config keys with a warning, so appending both is safe. https://claude.ai/code/session_014StKTjk7GBcVdiWKimEsQb --- noxfile.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/noxfile.py b/noxfile.py index ca7e4b6778ff..d472c55aed9b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -82,6 +82,25 @@ def pin_pyo3_config(session: nox.Session) -> None: break config_lines.append(line) assert config_lines, f"failed to extract PyO3 config from:\n{output}" + # maturin also reads PYO3_CONFIG_FILE, and additionally needs + # ext_suffix (mandatory on Windows) and abiflags (for the free-threaded + # wheel tag), neither of which PyO3 emits (it ignores unknown keys + # with a warning). + info = session.run_install( + str(python), + "-c", + "import sys, sysconfig;" + 'print(sysconfig.get_config_var("EXT_SUFFIX"));' + 'print(getattr(sys, "abiflags", "") or' + ' ("t" if sysconfig.get_config_var("Py_GIL_DISABLED") else ""))', + silent=True, + external=True, + ) + assert isinstance(info, str) + ext_suffix, abiflags = info.splitlines() + config_lines.append(f"ext_suffix={ext_suffix}") + if abiflags: + config_lines.append(f"abiflags={abiflags}") content = "\n".join(config_lines) + "\n" # The file name is content-addressed and the mtime is pinned: cargo # treats both a changed PYO3_CONFIG_FILE value and a fresh mtime on From f8a479309428735acf8167721b846f7d50a2dfb1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 05:28:49 +0000 Subject: [PATCH 3/3] Provide a usable PYO3_PYTHON alongside the pinned PyO3 config When PYO3_CONFIG_FILE is set, maturin doesn't export PYO3_PYTHON to cargo, so cryptography-cffi's build script fell back to 'python3' to run _cffi_src/build_openssl.py. On Windows that resolved to an interpreter without cffi (venvs there don't ship python3.exe), breaking all Windows jobs. Point PYO3_PYTHON at the session venv's interpreter -- a stable path, so no cache churn -- and install the build requirements into the session venv so that interpreter can run the cffi bridge script. The build itself still runs in an isolated environment. https://claude.ai/code/session_014StKTjk7GBcVdiWKimEsQb --- noxfile.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/noxfile.py b/noxfile.py index d472c55aed9b..153322afad05 100644 --- a/noxfile.py +++ b/noxfile.py @@ -112,6 +112,12 @@ def pin_pyo3_config(session: nox.Session) -> None: config_path.write_text(content) os.utime(config_path, (0, 0)) session.env["PYO3_CONFIG_FILE"] = str(config_path) + # When PYO3_CONFIG_FILE is set, maturin doesn't export PYO3_PYTHON to + # cargo, but cryptography-cffi's build script needs an interpreter + # (with the build requirements available) to run + # _cffi_src/build_openssl.py. Point it at the session's interpreter; + # the path is stable, so this doesn't reintroduce cache churn. + session.env["PYO3_PYTHON"] = str(python) @nox.session @@ -142,6 +148,12 @@ def tests(session: nox.Session) -> None: install_spec = f".[{','.join(extras)}]" pin_pyo3_config(session) + # The build requirements must be importable by the PYO3_PYTHON + # interpreter that pin_pyo3_config points at the session venv (see + # there); they are not otherwise used, since the build itself still + # runs in an isolated environment. + pyproject_data = load_pyproject_toml() + install(session, *pyproject_data["build-system"]["requires"]) install(session, "-e", "./vectors") if session.name == "tests-rust-debug": install(