diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index 8e729e66e4a9..ce61c3c2d708 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -19,4 +19,4 @@ runs: KEY: "${{ inputs.key }}" - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - key: ${{ steps.normalized-key.outputs.key }}-5 + key: ${{ steps.normalized-key.outputs.key }}-7 diff --git a/noxfile.py b/noxfile.py index aa34f95b406b..153322afad05 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,6 +5,7 @@ from __future__ import annotations import glob +import hashlib import itertools import json import os @@ -44,6 +45,81 @@ def load_pyproject_toml() -> dict: return tomllib.load(f) +def pin_pyo3_config(session: nox.Session) -> None: + # PEP 517 builds run in a randomly-named temporary environment, and + # PyO3's build script fingerprints the interpreter path it is given. + # That makes cargo recompile pyo3 (and everything downstream of it) on + # every CI run, despite an otherwise warm cache. Resolve the PyO3 + # build config once, against this session's interpreter (whose path is + # stable from run to run), and pin it via PYO3_CONFIG_FILE so the + # build is independent of the ephemeral build environment. + venv = pathlib.Path(session.virtualenv.location) + python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + output = session.run_install( + "cargo", + "check", + "-p", + "pyo3-ffi", + external=True, + silent=True, + success_codes=[0, 101], + env={"PYO3_PRINT_CONFIG": "1", "PYO3_PYTHON": str(python)}, + ) + # None on --no-install invocations, where the config file (and the + # build it is for) already exists. + if output is None: + return + assert isinstance(output, str) + config_lines = [] + in_config = False + for line in output.splitlines(): + line = line.strip() + if "PYO3_PRINT_CONFIG=1 is set" in line: + in_config = True + continue + if in_config: + if line.startswith("note:") or "=" not in line: + break + config_lines.append(line) + assert config_lines, f"failed to extract PyO3 config from:\n{output}" + # maturin also reads PYO3_CONFIG_FILE, and additionally needs + # ext_suffix (mandatory on Windows) and abiflags (for the free-threaded + # wheel tag), neither of which PyO3 emits (it ignores unknown keys + # with a warning). + info = session.run_install( + str(python), + "-c", + "import sys, sysconfig;" + 'print(sysconfig.get_config_var("EXT_SUFFIX"));' + 'print(getattr(sys, "abiflags", "") or' + ' ("t" if sysconfig.get_config_var("Py_GIL_DISABLED") else ""))', + silent=True, + external=True, + ) + assert isinstance(info, str) + ext_suffix, abiflags = info.splitlines() + config_lines.append(f"ext_suffix={ext_suffix}") + if abiflags: + config_lines.append(f"abiflags={abiflags}") + content = "\n".join(config_lines) + "\n" + # The file name is content-addressed and the mtime is pinned: cargo + # treats both a changed PYO3_CONFIG_FILE value and a fresh mtime on + # the file it points to as reasons to recompile pyo3. This way a + # regenerated-but-identical config (e.g. a fresh CI run) never looks + # changed, while a genuinely different config gets a new path. + digest = hashlib.sha256(content.encode()).hexdigest()[:16] + config_path = venv / f"pyo3-config-{digest}.txt" + config_path.write_text(content) + os.utime(config_path, (0, 0)) + session.env["PYO3_CONFIG_FILE"] = str(config_path) + # When PYO3_CONFIG_FILE is set, maturin doesn't export PYO3_PYTHON to + # cargo, but cryptography-cffi's build script needs an interpreter + # (with the build requirements available) to run + # _cffi_src/build_openssl.py. Point it at the session's interpreter; + # the path is stable, so this doesn't reintroduce cache churn. + session.env["PYO3_PYTHON"] = str(python) + + @nox.session @nox.session(name="tests-ssh") @nox.session(name="tests-randomorder") @@ -71,6 +147,13 @@ def tests(session: nox.Session) -> None: ) install_spec = f".[{','.join(extras)}]" + pin_pyo3_config(session) + # The build requirements must be importable by the PYO3_PYTHON + # interpreter that pin_pyo3_config points at the session venv (see + # there); they are not otherwise used, since the build itself still + # runs in an isolated environment. + pyproject_data = load_pyproject_toml() + install(session, *pyproject_data["build-system"]["requires"]) install(session, "-e", "./vectors") if session.name == "tests-rust-debug": install(