CI: stop recompiling pyo3 on every run (pin PYO3_CONFIG_FILE in tests sessions)

.github/actions/cache/action.yml

@@ -19,4 +19,4 @@ runs:
         KEY: "${{ inputs.key }}"
     - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4  # v2.9.1
       with:
-        key: ${{ steps.normalized-key.outputs.key }}-5
+        key: ${{ steps.normalized-key.outputs.key }}-7

noxfile.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import glob
+import hashlib
 import itertools
 import json
 import os
@@ -44,6 +45,81 @@ def load_pyproject_toml() -> dict:
         return tomllib.load(f)
 
 
+def pin_pyo3_config(session: nox.Session) -> None:
+    # PEP 517 builds run in a randomly-named temporary environment, and
+    # PyO3's build script fingerprints the interpreter path it is given.
+    # That makes cargo recompile pyo3 (and everything downstream of it) on
+    # every CI run, despite an otherwise warm cache. Resolve the PyO3
+    # build config once, against this session's interpreter (whose path is
+    # stable from run to run), and pin it via PYO3_CONFIG_FILE so the
+    # build is independent of the ephemeral build environment.
+    venv = pathlib.Path(session.virtualenv.location)
+    python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
+    output = session.run_install(
+        "cargo",
+        "check",
+        "-p",
+        "pyo3-ffi",
+        external=True,
+        silent=True,
+        success_codes=[0, 101],
+        env={"PYO3_PRINT_CONFIG": "1", "PYO3_PYTHON": str(python)},
+    )
+    # None on --no-install invocations, where the config file (and the
+    # build it is for) already exists.
+    if output is None:
+        return
+    assert isinstance(output, str)
+    config_lines = []
+    in_config = False
+    for line in output.splitlines():
+        line = line.strip()
+        if "PYO3_PRINT_CONFIG=1 is set" in line:
+            in_config = True
+            continue
+        if in_config:
+            if line.startswith("note:") or "=" not in line:
+                break
+            config_lines.append(line)
+    assert config_lines, f"failed to extract PyO3 config from:\n{output}"
+    # maturin also reads PYO3_CONFIG_FILE, and additionally needs
+    # ext_suffix (mandatory on Windows) and abiflags (for the free-threaded
+    # wheel tag), neither of which PyO3 emits (it ignores unknown keys
+    # with a warning).
+    info = session.run_install(
+        str(python),
+        "-c",
+        "import sys, sysconfig;"
+        'print(sysconfig.get_config_var("EXT_SUFFIX"));'
+        'print(getattr(sys, "abiflags", "") or'
+        ' ("t" if sysconfig.get_config_var("Py_GIL_DISABLED") else ""))',
+        silent=True,
+        external=True,
+    )
+    assert isinstance(info, str)
+    ext_suffix, abiflags = info.splitlines()
+    config_lines.append(f"ext_suffix={ext_suffix}")
+    if abiflags:
+        config_lines.append(f"abiflags={abiflags}")
+    content = "\n".join(config_lines) + "\n"
+    # The file name is content-addressed and the mtime is pinned: cargo
+    # treats both a changed PYO3_CONFIG_FILE value and a fresh mtime on
+    # the file it points to as reasons to recompile pyo3. This way a
+    # regenerated-but-identical config (e.g. a fresh CI run) never looks
+    # changed, while a genuinely different config gets a new path.
+    digest = hashlib.sha256(content.encode()).hexdigest()[:16]
+    config_path = venv / f"pyo3-config-{digest}.txt"
+    config_path.write_text(content)
+    os.utime(config_path, (0, 0))
+    session.env["PYO3_CONFIG_FILE"] = str(config_path)
+    # When PYO3_CONFIG_FILE is set, maturin doesn't export PYO3_PYTHON to
+    # cargo, but cryptography-cffi's build script needs an interpreter
+    # (with the build requirements available) to run
+    # _cffi_src/build_openssl.py. Point it at the session's interpreter;
+    # the path is stable, so this doesn't reintroduce cache churn.
+    session.env["PYO3_PYTHON"] = str(python)
+
+
 @nox.session
 @nox.session(name="tests-ssh")
 @nox.session(name="tests-randomorder")
@@ -71,6 +147,13 @@ def tests(session: nox.Session) -> None:
         )
 
     install_spec = f".[{','.join(extras)}]"
+    pin_pyo3_config(session)
+    # The build requirements must be importable by the PYO3_PYTHON
+    # interpreter that pin_pyo3_config points at the session venv (see
+    # there); they are not otherwise used, since the build itself still
+    # runs in an isolated environment.
+    pyproject_data = load_pyproject_toml()
+    install(session, *pyproject_data["build-system"]["requires"])
     install(session, "-e", "./vectors")
     if session.name == "tests-rust-debug":
         install(