diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 92db1410..45d0cee8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,6 +16,7 @@ on: - 'bt_api_py/**' - 'scripts/generate_exchange_support_docs.py' - 'scripts/ci/check_docs_contract.py' + - 'scripts/ci/base_source_pin.py' pull_request: paths: - 'docs/**' @@ -30,6 +31,7 @@ on: - 'bt_api_py/**' - 'scripts/generate_exchange_support_docs.py' - 'scripts/ci/check_docs_contract.py' + - 'scripts/ci/base_source_pin.py' workflow_dispatch: permissions: @@ -47,6 +49,9 @@ jobs: with: fetch-depth: 0 + - name: Checkout pinned base source + run: git submodule update --init --depth 1 -- bt_api/bt_api_base + - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -58,9 +63,17 @@ jobs: - name: Install doc dependencies run: python -m pip install --upgrade pip -r docs/requirements.txt + - name: Build and install parent-pinned base wheel + run: >- + python scripts/ci/base_source_pin.py + --wheel-dir "${{ runner.temp }}/bt_api_base_wheelhouse" + - name: Install package (needed by mkdocstrings) run: python -m pip install -e . + - name: Check installed dependencies + run: python -m pip check + - name: Check generated docs are current run: python scripts/generate_exchange_support_docs.py --check diff --git a/.github/workflows/reusable-compat-matrix.yml b/.github/workflows/reusable-compat-matrix.yml index 6bc343b3..0d757673 100644 --- a/.github/workflows/reusable-compat-matrix.yml +++ b/.github/workflows/reusable-compat-matrix.yml @@ -35,12 +35,20 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Checkout pinned base source + run: git submodule update --init --depth 1 -- bt_api/bt_api_base + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip cache-dependency-path: pyproject.toml + - name: Build and install parent-pinned base wheel + run: >- + python scripts/ci/base_source_pin.py + --wheel-dir "${{ runner.temp }}/bt_api_base_wheelhouse" + - name: Install build tools (Linux) if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y build-essential @@ -50,6 +58,9 @@ jobs: python -m pip install --upgrade pip pip install -e ".[dev]" + - name: Check installed dependencies + run: python -m pip check + - name: Run cross-platform compatibility smoke suite env: SKIP_LIVE_TESTS: "true" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 395fad01..3ebce44d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,12 +45,20 @@ jobs: cache: pip cache-dependency-path: pyproject.toml + - name: Build and install parent-pinned base wheel + run: >- + python scripts/ci/base_source_pin.py + --wheel-dir "${{ runner.temp }}/bt_api_base_wheelhouse" + - name: Install package + quality tools run: | python -m pip install --upgrade pip pip install -e ".[dev]" pip install bandit pip-audit + - name: Check installed dependencies + run: python -m pip check + - name: Ruff lint check run: ruff check bt_api_py tests --output-format=github @@ -159,8 +167,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 needs: quality - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} defaults: run: @@ -177,15 +183,38 @@ jobs: cache: pip cache-dependency-path: pyproject.toml + - name: Build and install parent-pinned base wheel + run: >- + python scripts/ci/base_source_pin.py + --wheel-dir "${{ runner.temp }}/bt_api_base_wheelhouse" + --github-env "$GITHUB_ENV" + - name: Install build tools run: sudo apt-get update && sudo apt-get install -y build-essential + - name: Build and install parent-pinned CTP source wheel + run: >- + python scripts/ci/ctp_source_pin.py + --wheel-dir "${{ runner.temp }}/bt_api_ctp_wheelhouse" + --github-env "$GITHUB_ENV" + - name: Install package + dev deps run: | python -m pip install --upgrade pip # Unmarked contract cases exercise the declared reference adapters. pip install -e ".[dev,security,core-reference]" -r requirements-ci-core-reference.txt + - name: Verify parent-pinned CTP wheel receipt + run: python scripts/ci/ctp_source_pin.py --verify-installed + + - name: Check installed dependencies + run: python -m pip check + + - name: Check CTP native extension load + run: >- + python -c "from bt_api_ctp.ctp._ctp_base import get_ctp_import_error, + is_ctp_native_loaded; assert is_ctp_native_loaded(), get_ctp_import_error()" + - name: Run baseline suite with coverage gate env: SKIP_LIVE_TESTS: "true" @@ -229,8 +258,13 @@ jobs: test -s htmlcov/index.html - name: Upload coverage to Codecov - if: always() && env.CODECOV_TOKEN != '' + if: >- + always() && + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository) uses: codecov/codecov-action@v6 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage-full.xml @@ -258,6 +292,9 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Checkout parent-pinned base source + run: git submodule update --init --depth 1 -- bt_api/bt_api_base + - uses: actions/setup-python@v6 with: python-version: "3.11" diff --git a/bt_api/bt_api_base b/bt_api/bt_api_base index 3de0fa4f..4413088e 160000 --- a/bt_api/bt_api_base +++ b/bt_api/bt_api_base @@ -1 +1 @@ -Subproject commit 3de0fa4f6cfe8d1973e9f4b9b47b01254259524f +Subproject commit 4413088e1bceaf14b6dbadda31e6e1b7c2a96631 diff --git a/bt_api/bt_api_ctp b/bt_api/bt_api_ctp index ce1edd60..4c10d18f 160000 --- a/bt_api/bt_api_ctp +++ b/bt_api/bt_api_ctp @@ -1 +1 @@ -Subproject commit ce1edd60785eb4c66fefa16a994a66946a1e068f +Subproject commit 4c10d18fa64b106449e28585dbca55e53dfe21e7 diff --git a/bt_api/bt_api_gateway b/bt_api/bt_api_gateway index 44fd2fe2..bf51a9bc 160000 --- a/bt_api/bt_api_gateway +++ b/bt_api/bt_api_gateway @@ -1 +1 @@ -Subproject commit 44fd2fe26b51f1d8b573c84415a6cff660f33dea +Subproject commit bf51a9bc31a128374af81968d44ccdfc4643ee40 diff --git a/bt_api/bt_api_monitor b/bt_api/bt_api_monitor index d515a820..246aae1f 160000 --- a/bt_api/bt_api_monitor +++ b/bt_api/bt_api_monitor @@ -1 +1 @@ -Subproject commit d515a8209324d56742c095d70976593bb3ba3eff +Subproject commit 246aae1f92e9e512e561320b7c53b9992a87b50c diff --git a/bt_api_py/configs/exchange-bundles.toml b/bt_api_py/configs/exchange-bundles.toml index 417798bb..2bcf252c 100644 --- a/bt_api_py/configs/exchange-bundles.toml +++ b/bt_api_py/configs/exchange-bundles.toml @@ -29,5 +29,5 @@ certification = "experimental" package = "bt_api_ctp" plugin = "ctp" exchange = "CTP___FUTURE" -min_version = "2.0.2" +min_version = "2.0.3" certification = "experimental" diff --git a/bt_api_py/forwarding/router.py b/bt_api_py/forwarding/router.py index 069c7904..b84a26cf 100644 --- a/bt_api_py/forwarding/router.py +++ b/bt_api_py/forwarding/router.py @@ -24,9 +24,7 @@ _VALID_SIDES = frozenset({"buy", "sell"}) _VALID_ORDER_TYPES = frozenset({"limit", "market"}) _MAX_CACHED_ACKS = 10_000 -_TRADING_DISABLED_REASON = ( - "forwarding trading is disabled; provider write was not attempted" -) +_TRADING_DISABLED_REASON = "forwarding trading is disabled; provider write was not attempted" @dataclass(frozen=True) diff --git a/docs/acceptance/2026-09-19-quality-ratchet.json b/docs/acceptance/2026-09-19-quality-ratchet.json index a1f9c361..082d5078 100644 --- a/docs/acceptance/2026-09-19-quality-ratchet.json +++ b/docs/acceptance/2026-09-19-quality-ratchet.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "generated_at": "2026-09-19T17:21:53.591707+00:00", + "generated_at": "2026-09-28T14:02:47.687472+00:00", "ruff_version": "ruff 0.16.2", "scope": [ "bt_api_py", @@ -15,13 +15,16 @@ "bt_api/bt_api_ctp/src", "bt_api/bt_api_dydx/src", "bt_api/bt_api_gateio/src", + "bt_api/bt_api_gateway/src", "bt_api/bt_api_htx/src", "bt_api/bt_api_hyperliquid/src", "bt_api/bt_api_ib_web/src", "bt_api/bt_api_kraken/src", "bt_api/bt_api_mexc/src", + "bt_api/bt_api_monitor/src", "bt_api/bt_api_mt5/src", "bt_api/bt_api_okx/src", + "bt_api/bt_api_risk/src", "bt_api/bt_api_base/tests", "bt_api/bt_api_binance/tests", "bt_api/bt_api_bitget/tests", @@ -30,13 +33,16 @@ "bt_api/bt_api_ctp/tests", "bt_api/bt_api_dydx/tests", "bt_api/bt_api_gateio/tests", + "bt_api/bt_api_gateway/tests", "bt_api/bt_api_htx/tests", "bt_api/bt_api_hyperliquid/tests", "bt_api/bt_api_ib_web/tests", "bt_api/bt_api_kraken/tests", "bt_api/bt_api_mexc/tests", + "bt_api/bt_api_monitor/tests", "bt_api/bt_api_mt5/tests", - "bt_api/bt_api_okx/tests" + "bt_api/bt_api_okx/tests", + "bt_api/bt_api_risk/tests" ], "ruff": { "total": 10, diff --git a/docs/acceptance/2026-09-20-format-ratchet.json b/docs/acceptance/2026-09-20-format-ratchet.json index b106f4ff..e1c3e8ee 100644 --- a/docs/acceptance/2026-09-20-format-ratchet.json +++ b/docs/acceptance/2026-09-20-format-ratchet.json @@ -14,14 +14,18 @@ "bt_api_coinbase", "bt_api_ctp", "bt_api_dydx", + "bt_api_execution", "bt_api_gateio", + "bt_api_gateway", "bt_api_htx", "bt_api_hyperliquid", "bt_api_ib_web", "bt_api_kraken", "bt_api_mexc", + "bt_api_monitor", "bt_api_mt5", - "bt_api_okx" + "bt_api_okx", + "bt_api_risk" ], "format": { "total": 451, @@ -33,14 +37,18 @@ "bt_api_coinbase": 23, "bt_api_ctp": 25, "bt_api_dydx": 20, + "bt_api_execution": 0, "bt_api_gateio": 25, + "bt_api_gateway": 0, "bt_api_htx": 23, "bt_api_hyperliquid": 23, "bt_api_ib_web": 40, "bt_api_kraken": 7, "bt_api_mexc": 12, + "bt_api_monitor": 0, "bt_api_mt5": 7, - "bt_api_okx": 85 + "bt_api_okx": 85, + "bt_api_risk": 0 } }, "notes": "Submodule format ratchet baseline. Per-module counts may only decrease; use --update after improvements or --force-update for an intentional rebaseline." diff --git a/pyproject.toml b/pyproject.toml index adcdee41..ceb0cf9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ authors = [ {name = "cloudQuant", email = "yunjinqi@gmail.com"}, ] dependencies = [ - "bt_api_base>=0.15.4", + "bt_api_base>=0.15.5", "numpy>=1.26.0", "python-dotenv>=1.0.0", "requests>=2.31.0", @@ -303,5 +303,5 @@ all = [ ] core-reference = [ "bt_api_binance>=2.0.0", - "bt_api_ctp>=2.0.2,<3.0", + "bt_api_ctp>=2.0.3,<3.0", ] diff --git a/scripts/ci/base_source_pin.py b/scripts/ci/base_source_pin.py new file mode 100644 index 00000000..9c45695a --- /dev/null +++ b/scripts/ci/base_source_pin.py @@ -0,0 +1,490 @@ +#!/usr/bin/env python3 +"""Build/install the pinned base source for CI (Python 3.11 through 3.14).""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import re +import subprocess +import sys +import tarfile +import tempfile +import tomllib +import zipfile +from dataclasses import asdict, dataclass +from email.parser import BytesParser +from pathlib import Path, PurePosixPath + +if __package__: + from .offline_pip import pip_source_args, resolve_wheelhouse_path +else: + from offline_pip import pip_source_args, resolve_wheelhouse_path + +BASE_PATH = "bt_api/bt_api_base" +BASE_ORIGIN = "https://github.com/cloudQuant/bt_api_base.git" +MINIMUM_BASE_VERSION = (0, 15, 5) +GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +STABLE_VERSION_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)$") +BASE_REQUIREMENT_RE = re.compile(r"^bt_api_base\s*>=\s*(\d+\.\d+\.\d+)$", re.IGNORECASE) +SUBPROCESS_ENV_ALLOWLIST = ( + "PATH", + "SystemRoot", + "WINDIR", + "COMSPEC", + "PATHEXT", + "TEMP", + "TMP", + "TMPDIR", + "HOME", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "LANG", + "LC_ALL", + "LC_CTYPE", +) + + +class BaseSourcePinError(RuntimeError): + """Raised when the checked-out source cannot be bound to the parent gitlink.""" + + +@dataclass(frozen=True) +class BaseSourcePin: + parent_commit: str + source_commit: str + source_tree: str + source_origin: str + package_name: str + package_version: str + minimum_version: str + source_path: Path + + +@dataclass(frozen=True) +class BaseWheelReceipt: + parent_commit: str + source_commit: str + source_tree: str + source_origin: str + package_name: str + package_version: str + minimum_version: str + wheel_filename: str + wheel_sha256: str + wheel_path: Path + + +def _subprocess_environment(*, public_pip_index: bool = False) -> dict[str, str]: + """Pass only OS runtime settings and the explicit public pip configuration.""" + environment = { + name: os.environ[name] for name in SUBPROCESS_ENV_ALLOWLIST if name in os.environ + } + environment["PIP_CONFIG_FILE"] = os.devnull + if public_pip_index: + environment["PIP_INDEX_URL"] = "https://pypi.org/simple/" + return environment + + +def _run_git(repository: Path, *arguments: str) -> str: + completed = subprocess.run( # noqa: S603 + ["git", "-C", str(repository), *arguments], # noqa: S607 + capture_output=True, + check=False, + text=True, + env=_subprocess_environment(), + ) + if completed.returncode: + command = arguments[0] if arguments else "unknown" + raise BaseSourcePinError(f"git {command!r} failed with exit code {completed.returncode}") + return completed.stdout.strip() + + +def _stable_version(version: str, *, label: str) -> tuple[int, int, int]: + match = STABLE_VERSION_RE.fullmatch(version) + if not match: + raise BaseSourcePinError(f"{label} must be a stable three-part version: {version!r}") + return tuple(int(part) for part in match.groups()) # type: ignore[return-value] + + +def _source_metadata(source_path: Path, source_commit: str) -> tuple[str, str]: + result = subprocess.run( # noqa: S603 + [ # noqa: S607 + "git", + "-C", + str(source_path), + "show", + f"{source_commit}:pyproject.toml", + ], + capture_output=True, + check=False, + env=_subprocess_environment(), + ) + if result.returncode: + raise BaseSourcePinError("pinned bt_api_base source has no readable pyproject.toml") + try: + data = tomllib.loads(result.stdout.decode("utf-8")) + except (UnicodeDecodeError, tomllib.TOMLDecodeError) as exc: + raise BaseSourcePinError(f"pinned bt_api_base pyproject.toml is invalid: {exc}") from exc + project = data.get("project") + if not isinstance(project, dict): + raise BaseSourcePinError("pinned bt_api_base pyproject.toml has no [project] table") + name = str(project.get("name") or "") + version = str(project.get("version") or "") + if name != "bt_api_base": + raise BaseSourcePinError(f"pinned source package name must be bt_api_base, got {name!r}") + _stable_version(version, label="pinned bt_api_base version") + return name, version + + +def verify_base_source_pin(repository_root: Path) -> BaseSourcePin: + """Bind the source checkout to the full gitlink SHA and canonical upstream URL.""" + repository_root = repository_root.resolve(strict=True) + parent_commit = _run_git(repository_root, "rev-parse", "HEAD") + tree_entry = _run_git(repository_root, "ls-tree", parent_commit, "--", BASE_PATH) + fields = tree_entry.split() + if len(fields) != 4 or fields[0] != "160000" or fields[1] != "commit" or fields[3] != BASE_PATH: + raise BaseSourcePinError(f"{BASE_PATH} is not an exact gitlink in parent HEAD") + source_commit = fields[2] + if not GIT_SHA_RE.fullmatch(source_commit): + raise BaseSourcePinError( + f"parent gitlink is not a full 40-character SHA: {source_commit!r}" + ) + + source_path = repository_root / BASE_PATH + if not source_path.is_dir(): + raise BaseSourcePinError(f"pinned source checkout is unavailable: {source_path}") + + configured_origin = _run_git( + repository_root, + "config", + "-f", + ".gitmodules", + "--get", + "submodule.bt_api/bt_api_base.url", + ) + actual_origin = _run_git(source_path, "remote", "get-url", "origin") + if configured_origin != BASE_ORIGIN or actual_origin != BASE_ORIGIN: + raise BaseSourcePinError( + "bt_api_base origin mismatch: .gitmodules URL and source origin must both match " + f"canonical upstream {BASE_ORIGIN}" + ) + + checked_out_commit = _run_git(source_path, "rev-parse", "HEAD") + if checked_out_commit != source_commit: + raise BaseSourcePinError( + f"bt_api_base checkout {checked_out_commit} does not match parent gitlink {source_commit}" + ) + source_tree = _run_git(source_path, "rev-parse", f"{source_commit}^{{tree}}") + + root_pyproject = repository_root / "pyproject.toml" + try: + root_data = tomllib.loads(root_pyproject.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise BaseSourcePinError(f"cannot read parent pyproject.toml: {exc}") from exc + dependencies = root_data.get("project", {}).get("dependencies", []) + matches = [ + match + for requirement in dependencies + if (match := BASE_REQUIREMENT_RE.fullmatch(str(requirement).strip())) is not None + ] + if len(matches) != 1: + raise BaseSourcePinError( + "parent dependencies must contain exactly one simple bt_api_base>=X.Y.Z requirement" + ) + minimum_version = matches[0].group(1) + minimum = _stable_version(minimum_version, label="parent bt_api_base minimum version") + if minimum < MINIMUM_BASE_VERSION: + raise BaseSourcePinError( + f"parent bt_api_base lower bound must remain >=0.15.5, got {minimum_version}" + ) + + package_name, package_version = _source_metadata(source_path, source_commit) + version_tuple = _stable_version(package_version, label="pinned bt_api_base version") + if version_tuple < minimum: + raise BaseSourcePinError( + f"pinned bt_api_base {package_version} does not satisfy parent >= {minimum_version}" + ) + + return BaseSourcePin( + parent_commit=parent_commit, + source_commit=source_commit, + source_tree=source_tree, + source_origin=actual_origin, + package_name=package_name, + package_version=package_version, + minimum_version=minimum_version, + source_path=source_path, + ) + + +def _extract_pinned_archive(pin: BaseSourcePin, destination: Path) -> Path: + completed = subprocess.run( # noqa: S603 + [ # noqa: S607 + "git", + "-C", + str(pin.source_path), + "archive", + "--format=tar", + pin.source_commit, + ], + capture_output=True, + check=False, + env=_subprocess_environment(), + ) + if completed.returncode: + raise BaseSourcePinError( + f"cannot archive pinned bt_api_base source (git exit code {completed.returncode})" + ) + + source_root = destination / "source" + source_root.mkdir(parents=True) + try: + with tarfile.open(fileobj=io.BytesIO(completed.stdout), mode="r:") as archive: + for member in archive.getmembers(): + relative = PurePosixPath(member.name) + if ( + relative.is_absolute() + or ".." in relative.parts + or "\\" in member.name + or ":" in member.name + ): + raise BaseSourcePinError("pinned source archive contains an unsafe path") + target = source_root.joinpath(*relative.parts) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + target.parent.mkdir(parents=True, exist_ok=True) + fileobj = archive.extractfile(member) + if fileobj is None: + raise BaseSourcePinError( + "pinned source archive contains an unreadable file" + ) + with fileobj, target.open("wb") as output: + output.write(fileobj.read()) + if member.mode & 0o111: + target.chmod(target.stat().st_mode | 0o111) + else: + raise BaseSourcePinError( + "pinned source archive contains a non-regular entry; refusing extraction" + ) + except tarfile.TarError as exc: + raise BaseSourcePinError(f"pinned source archive is invalid: {exc}") from exc + return source_root + + +def _read_wheel_identity(wheel_path: Path) -> tuple[str, str]: + try: + with zipfile.ZipFile(wheel_path) as archive: + metadata_paths = [ + name for name in archive.namelist() if name.endswith(".dist-info/METADATA") + ] + if len(metadata_paths) != 1: + raise BaseSourcePinError( + f"expected one wheel METADATA file, found {len(metadata_paths)}" + ) + metadata = BytesParser().parsebytes(archive.read(metadata_paths[0])) + except (OSError, zipfile.BadZipFile) as exc: + raise BaseSourcePinError( + f"pinned source build did not produce a valid wheel: {exc}" + ) from exc + return str(metadata.get("Name") or ""), str(metadata.get("Version") or "") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _safe_env_value(value: str) -> str: + if "\n" in value or "\r" in value: + raise BaseSourcePinError("refusing multiline GitHub environment value") + return value + + +def _verify_installed_wheel(receipt: BaseWheelReceipt) -> None: + try: + from importlib import metadata + except ImportError as exc: # pragma: no cover - Python 3.11+ provides this module + raise BaseSourcePinError(f"cannot import installed distribution metadata: {exc}") from exc + + try: + distribution = metadata.distribution(receipt.package_name) + direct_url_text = distribution.read_text("direct_url.json") + direct_url = json.loads(direct_url_text) if direct_url_text else {} + except (ImportError, json.JSONDecodeError, metadata.PackageNotFoundError) as exc: + raise BaseSourcePinError(f"cannot verify installed source wheel receipt: {exc}") from exc + + if distribution.version != receipt.package_version: + raise BaseSourcePinError( + f"installed bt_api_base {distribution.version} does not match pinned wheel " + f"{receipt.package_version}" + ) + archive_info = direct_url.get("archive_info") or {} + recorded_hash = archive_info.get("hash") + if recorded_hash is None: + recorded_hash = (archive_info.get("hashes") or {}).get("sha256") + if recorded_hash: + recorded_hash = f"sha256={recorded_hash}" + if recorded_hash != f"sha256={receipt.wheel_sha256}": + raise BaseSourcePinError( + "installed bt_api_base direct_url.json does not match the local wheel SHA-256" + ) + + +def build_pinned_base_wheel( + repository_root: Path, + wheel_dir: Path, + *, + python: str = sys.executable, + build_wheelhouse: Path | None = None, +) -> BaseWheelReceipt: + """Build a wheel from the parent-pinned source and return its provenance receipt.""" + pin = verify_base_source_pin(repository_root) + build_wheelhouse = resolve_wheelhouse_path(build_wheelhouse) + wheel_dir = wheel_dir.resolve() + wheel_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="bt-api-base-pinned-source-") as temporary: + temporary_root = Path(temporary) + source_root = _extract_pinned_archive(pin, temporary_root) + output_dir = temporary_root / "wheel-output" + output_dir.mkdir() + environment = _subprocess_environment(public_pip_index=build_wheelhouse is None) + command = [ + python, + "-m", + "pip", + "wheel", + "--no-deps", + *pip_source_args(build_wheelhouse), + "--wheel-dir", + str(output_dir), + str(source_root), + ] + completed = subprocess.run( # noqa: S603 + command, + capture_output=True, + check=False, + text=True, + env=environment, + ) + if completed.returncode: + detail = completed.stderr.strip() or completed.stdout.strip() + raise BaseSourcePinError(f"pinned bt_api_base wheel build failed: {detail}") + + wheels = sorted(output_dir.glob("*.whl")) + if len(wheels) != 1: + raise BaseSourcePinError(f"expected one pinned bt_api_base wheel, found {len(wheels)}") + built_wheel = wheels[0] + wheel_name, wheel_version = _read_wheel_identity(built_wheel) + if wheel_name != pin.package_name or wheel_version != pin.package_version: + raise BaseSourcePinError( + f"pinned wheel metadata mismatch: Name={wheel_name!r}, Version={wheel_version!r}" + ) + final_wheel = wheel_dir / built_wheel.name + final_wheel.write_bytes(built_wheel.read_bytes()) + + wheel_name, wheel_version = _read_wheel_identity(final_wheel) + if wheel_name != pin.package_name or wheel_version != pin.package_version: + raise BaseSourcePinError("copied local wheel metadata changed after the source build") + wheel_sha256 = _sha256(final_wheel) + receipt = BaseWheelReceipt( + parent_commit=pin.parent_commit, + source_commit=pin.source_commit, + source_tree=pin.source_tree, + source_origin=pin.source_origin, + package_name=pin.package_name, + package_version=pin.package_version, + minimum_version=pin.minimum_version, + wheel_filename=final_wheel.name, + wheel_sha256=wheel_sha256, + wheel_path=final_wheel, + ) + + if _sha256(final_wheel) != receipt.wheel_sha256: + raise BaseSourcePinError("local bt_api_base wheel hash changed after the source build") + return receipt + + +def build_and_install_base_wheel( + repository_root: Path, + wheel_dir: Path, + *, + python: str = sys.executable, +) -> BaseWheelReceipt: + """Build the parent-pinned source archive, hash it, install it, and verify PEP 610.""" + receipt = build_pinned_base_wheel(repository_root, wheel_dir, python=python) + if _sha256(receipt.wheel_path) != receipt.wheel_sha256: + raise BaseSourcePinError("local bt_api_base wheel hash changed before installation") + install_environment = _subprocess_environment() + completed = subprocess.run( # noqa: S603 + [ + python, + "-m", + "pip", + "install", + "--no-deps", + "--force-reinstall", + str(receipt.wheel_path), + ], + capture_output=True, + check=False, + text=True, + env=install_environment, + ) + if completed.returncode: + detail = completed.stderr.strip() or completed.stdout.strip() + raise BaseSourcePinError(f"installing pinned bt_api_base wheel failed: {detail}") + _verify_installed_wheel(receipt) + return receipt + + +def _write_github_environment(path: Path, receipt: BaseWheelReceipt) -> None: + values = { + "BT_API_BASE_SOURCE_SHA": receipt.source_commit, + "BT_API_BASE_SOURCE_TREE_SHA": receipt.source_tree, + "BT_API_BASE_SOURCE_ORIGIN": receipt.source_origin, + "BT_API_BASE_SOURCE_VERSION": receipt.package_version, + "BT_API_BASE_WHEEL_SHA256": receipt.wheel_sha256, + "BT_API_BASE_WHEEL_PATH": str(receipt.wheel_path), + } + with path.open("a", encoding="utf-8", newline="\n") as handle: + for key, value in values.items(): + handle.write(f"{key}={_safe_env_value(value)}\n") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--repository-root", + type=Path, + default=Path(__file__).resolve().parents[2], + ) + parser.add_argument("--wheel-dir", type=Path, required=True) + parser.add_argument("--github-env", type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + receipt = build_and_install_base_wheel(args.repository_root, args.wheel_dir) + if args.github_env: + _write_github_environment(args.github_env, receipt) + except (BaseSourcePinError, OSError) as exc: + print(f"base source pin bootstrap failed: {exc}", file=sys.stderr) + return 1 + print(json.dumps(asdict(receipt), default=str, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/check_format_ratchet.py b/scripts/ci/check_format_ratchet.py index 36e42e1f..c311052f 100644 --- a/scripts/ci/check_format_ratchet.py +++ b/scripts/ci/check_format_ratchet.py @@ -131,6 +131,17 @@ def scan_module(module: str) -> int: raise FormatScanError(f"invalid submodule name: {module!r}") path = f"bt_api/{module}" result = _ruff("format", "--check", path) + # Ruff exits successfully with this warning for a checked-out adapter + # module that has no Python sources yet (for example, execution). Treat + # only this exact stderr-only response as zero debt; all other empty or + # unparseable output remains a fail-closed scan error. + if ( + result.returncode == 0 + and not (result.stdout or "").strip() + and (result.stderr or "").splitlines() + == ["warning: No Python files found under the given path(s)"] + ): + return 0 try: return parse_format_output(result.stdout or "", result.returncode) except FormatScanError as error: diff --git a/scripts/ci/ctp_source_pin.py b/scripts/ci/ctp_source_pin.py new file mode 100644 index 00000000..5c06c691 --- /dev/null +++ b/scripts/ci/ctp_source_pin.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 +"""Build and install the exact parent-pinned CTP source wheel for CI only.""" + +from __future__ import annotations + +import argparse +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import tomllib +import zipfile +from dataclasses import asdict, dataclass +from email.parser import BytesParser +from importlib import metadata +from pathlib import Path, PurePosixPath +from urllib.parse import unquote, urlparse + +if __package__: + from .base_source_pin import ( + BaseSourcePinError, + _run_git, + _safe_env_value, + _sha256, + _stable_version, + _subprocess_environment, + verify_base_source_pin, + ) +else: + from base_source_pin import ( + BaseSourcePinError, + _run_git, + _safe_env_value, + _sha256, + _stable_version, + _subprocess_environment, + verify_base_source_pin, + ) + +CTP_PATH = "bt_api/bt_api_ctp" +CTP_ORIGIN = "https://github.com/cloudQuant/bt_api_ctp.git" +CTP_MINIMUM_VERSION = (2, 0, 3) +BASE_MINIMUM_VERSION = (0, 15, 5) +GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +STABLE_VERSION_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)$") +CTP_REQUIREMENT_RE = re.compile( + r"^bt_api_ctp\s*>=\s*(\d+\.\d+\.\d+)\s*,\s*<\s*(\d+\.\d+)$", + re.IGNORECASE, +) +BASE_REQUIREMENT_RE = re.compile( + r"^bt_api_base\s*>=\s*(\d+\.\d+\.\d+)\s*,\s*<\s*(\d+\.\d+)$", + re.IGNORECASE, +) + + +class CtpSourcePinError(RuntimeError): + """Raised when the CTP wheel cannot be bound to the parent source pin.""" + + +@dataclass(frozen=True) +class CtpSourcePin: + parent_commit: str + source_commit: str + source_tree: str + source_origin: str + package_name: str + package_version: str + minimum_version: str + minimum_base_version: str + source_path: Path + + +@dataclass(frozen=True) +class CtpWheelReceipt: + parent_commit: str + source_commit: str + source_tree: str + source_origin: str + package_name: str + package_version: str + minimum_version: str + minimum_base_version: str + wheel_filename: str + wheel_sha256: str + wheel_path: Path + + +def _git_executable() -> str: + executable = shutil.which("git") + if executable is None: + raise CtpSourcePinError("git executable is unavailable in the allowlisted PATH") + return executable + + +def _source_metadata(source_path: Path, source_commit: str) -> tuple[str, str, str]: + result = subprocess.run( # noqa: S603 + [ + _git_executable(), + "-C", + str(source_path), + "show", + f"{source_commit}:pyproject.toml", + ], + capture_output=True, + check=False, + env=_subprocess_environment(), + ) + if result.returncode: + raise CtpSourcePinError("pinned bt_api_ctp source has no readable pyproject.toml") + try: + data = tomllib.loads(result.stdout.decode("utf-8")) + except (UnicodeDecodeError, tomllib.TOMLDecodeError) as exc: + raise CtpSourcePinError(f"pinned bt_api_ctp pyproject.toml is invalid: {exc}") from exc + + project = data.get("project") + if not isinstance(project, dict): + raise CtpSourcePinError("pinned bt_api_ctp pyproject.toml has no [project] table") + package_name = str(project.get("name") or "") + package_version = str(project.get("version") or "") + if package_name != "bt_api_ctp": + raise CtpSourcePinError( + f"pinned source package name must be bt_api_ctp, got {package_name!r}" + ) + _stable_version(package_version, label="pinned bt_api_ctp version") + + dependencies = project.get("dependencies", []) + matches = [ + match + for dependency in dependencies + if (match := BASE_REQUIREMENT_RE.fullmatch(str(dependency).strip())) is not None + ] + if len(matches) != 1: + raise CtpSourcePinError( + "pinned bt_api_ctp must declare exactly one simple bt_api_base>=X.Y.Z,=0.15.5, got {base_minimum}" + ) + return package_name, package_version, base_minimum + + +def _parent_ctp_requirement(repository_root: Path) -> str: + try: + root_data = tomllib.loads((repository_root / "pyproject.toml").read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise CtpSourcePinError(f"cannot read parent pyproject.toml: {exc}") from exc + optional = root_data.get("project", {}).get("optional-dependencies", {}) + core_reference = optional.get("core-reference", []) if isinstance(optional, dict) else [] + matches = [ + match + for requirement in core_reference + if (match := CTP_REQUIREMENT_RE.fullmatch(str(requirement).strip())) is not None + ] + if len(matches) != 1: + raise CtpSourcePinError( + "parent core-reference extra must contain exactly one simple " + "bt_api_ctp>=X.Y.Z,=2.0.3, got {minimum_version}" + ) + return minimum_version + + +def verify_ctp_source_pin(repository_root: Path) -> CtpSourcePin: + """Bind CTP metadata and wheel input to the exact parent gitlink.""" + repository_root = repository_root.resolve(strict=True) + parent_commit = _run_git(repository_root, "rev-parse", "HEAD") + tree_entry = _run_git(repository_root, "ls-tree", parent_commit, "--", CTP_PATH) + fields = tree_entry.split() + if len(fields) != 4 or fields[0] != "160000" or fields[1] != "commit" or fields[3] != CTP_PATH: + raise CtpSourcePinError(f"{CTP_PATH} is not an exact gitlink in parent HEAD") + source_commit = fields[2] + if not GIT_SHA_RE.fullmatch(source_commit): + raise CtpSourcePinError( + f"parent CTP gitlink is not a full 40-character SHA: {source_commit!r}" + ) + + source_path = repository_root / CTP_PATH + if not source_path.is_dir(): + raise CtpSourcePinError(f"pinned source checkout is unavailable: {source_path}") + configured_origin = _run_git( + repository_root, + "config", + "-f", + ".gitmodules", + "--get", + "submodule.bt_api/bt_api_ctp.url", + ) + actual_origin = _run_git(source_path, "remote", "get-url", "origin") + if configured_origin != CTP_ORIGIN or actual_origin != CTP_ORIGIN: + raise CtpSourcePinError( + "bt_api_ctp origin mismatch: .gitmodules URL and source origin must both match " + f"canonical upstream {CTP_ORIGIN}" + ) + checked_out_commit = _run_git(source_path, "rev-parse", "HEAD") + if checked_out_commit != source_commit: + raise CtpSourcePinError( + f"bt_api_ctp checkout {checked_out_commit} does not match parent gitlink {source_commit}" + ) + source_tree = _run_git(source_path, "rev-parse", f"{source_commit}^{{tree}}") + + package_name, package_version, minimum_base_version = _source_metadata( + source_path, source_commit + ) + package_version_tuple = _stable_version(package_version, label="pinned bt_api_ctp version") + if package_version_tuple < CTP_MINIMUM_VERSION: + raise CtpSourcePinError(f"pinned bt_api_ctp version must be >=2.0.3, got {package_version}") + minimum_version = _parent_ctp_requirement(repository_root) + if package_version_tuple < _stable_version(minimum_version, label="parent CTP minimum"): + raise CtpSourcePinError( + f"pinned bt_api_ctp {package_version} does not satisfy parent >= {minimum_version}" + ) + + bundle_path = repository_root / "bt_api_py" / "configs" / "exchange-bundles.toml" + try: + bundle_data = tomllib.loads(bundle_path.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise CtpSourcePinError(f"cannot read core-reference bundle metadata: {exc}") from exc + venues = bundle_data.get("bundles", {}).get("core-reference", {}).get("venues", []) + ctp_venues = [venue for venue in venues if venue.get("package") == "bt_api_ctp"] + if len(ctp_venues) != 1: + raise CtpSourcePinError("core-reference bundle must contain exactly one bt_api_ctp entry") + bundle_minimum = str(ctp_venues[0].get("min_version") or "") + if not STABLE_VERSION_RE.fullmatch(bundle_minimum): + raise CtpSourcePinError("core-reference bt_api_ctp min_version must be a stable X.Y.Z") + if package_version_tuple < _stable_version(bundle_minimum, label="bundle CTP minimum"): + raise CtpSourcePinError( + f"pinned bt_api_ctp {package_version} does not satisfy bundle >= {bundle_minimum}" + ) + if not (source_path / "setup.py").is_file(): + raise CtpSourcePinError( + "pinned bt_api_ctp source is missing setup.py native extension build" + ) + + return CtpSourcePin( + parent_commit=parent_commit, + source_commit=source_commit, + source_tree=source_tree, + source_origin=actual_origin, + package_name=package_name, + package_version=package_version, + minimum_version=minimum_version, + minimum_base_version=minimum_base_version, + source_path=source_path, + ) + + +def _extract_pinned_archive(pin: CtpSourcePin, destination: Path) -> Path: + completed = subprocess.run( # noqa: S603 + [ + _git_executable(), + "-C", + str(pin.source_path), + "archive", + "--format=tar", + pin.source_commit, + ], + capture_output=True, + check=False, + env=_subprocess_environment(), + ) + if completed.returncode: + raise CtpSourcePinError( + f"cannot archive pinned bt_api_ctp source (git exit code {completed.returncode})" + ) + source_root = destination / "source" + source_root.mkdir(parents=True) + try: + with tarfile.open(fileobj=io.BytesIO(completed.stdout), mode="r:") as archive: + for member in archive.getmembers(): + relative = PurePosixPath(member.name) + if ( + relative.is_absolute() + or ".." in relative.parts + or "\\" in member.name + or ":" in member.name + ): + raise CtpSourcePinError("pinned CTP source archive contains an unsafe path") + target = source_root.joinpath(*relative.parts) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + target.parent.mkdir(parents=True, exist_ok=True) + fileobj = archive.extractfile(member) + if fileobj is None: + raise CtpSourcePinError("pinned CTP archive contains an unreadable file") + with fileobj, target.open("wb") as output: + output.write(fileobj.read()) + if member.mode & 0o111: + target.chmod(target.stat().st_mode | 0o111) + else: + raise CtpSourcePinError( + "pinned CTP archive contains a non-regular entry; refusing extraction" + ) + except tarfile.TarError as exc: + raise CtpSourcePinError(f"pinned CTP source archive is invalid: {exc}") from exc + return source_root + + +def _read_wheel_identity(wheel_path: Path) -> tuple[str, str]: + try: + with zipfile.ZipFile(wheel_path) as archive: + metadata_paths = [ + name for name in archive.namelist() if name.endswith(".dist-info/METADATA") + ] + if len(metadata_paths) != 1: + raise CtpSourcePinError( + f"expected one wheel METADATA file, found {len(metadata_paths)}" + ) + metadata = BytesParser().parsebytes(archive.read(metadata_paths[0])) + except (OSError, zipfile.BadZipFile) as exc: + raise CtpSourcePinError(f"pinned CTP build did not produce a valid wheel: {exc}") from exc + return str(metadata.get("Name") or ""), str(metadata.get("Version") or "") + + +def _verified_archive_hash(direct_url: dict[str, object]) -> str: + archive_info = direct_url.get("archive_info") + if not isinstance(archive_info, dict): + return "" + recorded_hash = archive_info.get("hash") + if recorded_hash is None: + hashes = archive_info.get("hashes") + recorded_hash = hashes.get("sha256") if isinstance(hashes, dict) else None + if recorded_hash: + recorded_hash = f"sha256={recorded_hash}" + return str(recorded_hash or "") + + +def _verify_local_wheel_origin( + package_name: str, + package_version: str, + wheel_path: Path, + wheel_sha256: str, +) -> None: + try: + distribution = metadata.distribution(package_name) + direct_url_text = distribution.read_text("direct_url.json") + direct_url = json.loads(direct_url_text) if direct_url_text else {} + except (ImportError, json.JSONDecodeError, metadata.PackageNotFoundError) as exc: + raise CtpSourcePinError( + f"cannot verify installed {package_name} wheel origin: {exc}" + ) from exc + if distribution.version != package_version: + raise CtpSourcePinError( + f"installed {package_name} {distribution.version} does not match local wheel " + f"{package_version}" + ) + parsed_url = urlparse(str(direct_url.get("url") or "")) + if parsed_url.scheme != "file": + raise CtpSourcePinError(f"installed {package_name} has no local file PEP 610 origin") + url_path = unquote(parsed_url.path) + if os.name == "nt" and re.match(r"^/[A-Za-z]:/", url_path): + url_path = url_path[1:] + recorded_path = Path(url_path).resolve() + if recorded_path != wheel_path.resolve(): + raise CtpSourcePinError( + f"installed {package_name} PEP 610 URL is not the pinned local wheel" + ) + if _verified_archive_hash(direct_url) != f"sha256={wheel_sha256}": + raise CtpSourcePinError( + f"installed {package_name} direct_url.json does not match local wheel SHA-256" + ) + + +def _verify_base_receipt(repository_root: Path) -> None: + base_pin = verify_base_source_pin(repository_root) + expected = { + "BT_API_BASE_SOURCE_SHA": base_pin.source_commit, + "BT_API_BASE_SOURCE_TREE_SHA": base_pin.source_tree, + "BT_API_BASE_SOURCE_ORIGIN": base_pin.source_origin, + "BT_API_BASE_SOURCE_VERSION": base_pin.package_version, + } + for name, value in expected.items(): + if os.environ.get(name) != value: + raise CtpSourcePinError(f"installed base receipt {name} does not match parent gitlink") + wheel_path_text = os.environ.get("BT_API_BASE_WHEEL_PATH", "") + wheel_sha256 = os.environ.get("BT_API_BASE_WHEEL_SHA256", "") + if not wheel_path_text or not wheel_sha256: + raise CtpSourcePinError( + "pinned base wheel path/SHA receipt is missing from the environment" + ) + wheel_path = Path(wheel_path_text).resolve(strict=True) + if _sha256(wheel_path) != wheel_sha256: + raise CtpSourcePinError("pinned base wheel changed after source-pin installation") + _verify_local_wheel_origin("bt_api_base", base_pin.package_version, wheel_path, wheel_sha256) + + +def _verify_installed_ctp_receipt(repository_root: Path) -> CtpWheelReceipt: + pin = verify_ctp_source_pin(repository_root) + expected = { + "BT_API_CTP_SOURCE_SHA": pin.source_commit, + "BT_API_CTP_SOURCE_TREE_SHA": pin.source_tree, + "BT_API_CTP_SOURCE_ORIGIN": pin.source_origin, + "BT_API_CTP_SOURCE_VERSION": pin.package_version, + } + for name, value in expected.items(): + if os.environ.get(name) != value: + raise CtpSourcePinError(f"installed CTP receipt {name} does not match parent gitlink") + wheel_path_text = os.environ.get("BT_API_CTP_WHEEL_PATH", "") + wheel_sha256 = os.environ.get("BT_API_CTP_WHEEL_SHA256", "") + if not wheel_path_text or not wheel_sha256: + raise CtpSourcePinError("pinned CTP wheel path/SHA receipt is missing from the environment") + wheel_path = Path(wheel_path_text).resolve(strict=True) + if _sha256(wheel_path) != wheel_sha256: + raise CtpSourcePinError("pinned CTP wheel changed after source-pin installation") + wheel_name, wheel_version = _read_wheel_identity(wheel_path) + if wheel_name != pin.package_name or wheel_version != pin.package_version: + raise CtpSourcePinError("pinned CTP wheel METADATA no longer matches exact source metadata") + _verify_local_wheel_origin(pin.package_name, pin.package_version, wheel_path, wheel_sha256) + return CtpWheelReceipt( + parent_commit=pin.parent_commit, + source_commit=pin.source_commit, + source_tree=pin.source_tree, + source_origin=pin.source_origin, + package_name=pin.package_name, + package_version=pin.package_version, + minimum_version=pin.minimum_version, + minimum_base_version=pin.minimum_base_version, + wheel_filename=wheel_path.name, + wheel_sha256=wheel_sha256, + wheel_path=wheel_path, + ) + + +def build_and_install_ctp_wheel( + repository_root: Path, + wheel_dir: Path, + *, + python: str = sys.executable, +) -> CtpWheelReceipt: + """Build from a safe archive of the exact gitlink, install, then verify PEP 610.""" + repository_root = repository_root.resolve(strict=True) + pin = verify_ctp_source_pin(repository_root) + _verify_base_receipt(repository_root) + wheel_dir = wheel_dir.resolve() + wheel_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="bt-api-ctp-pinned-source-") as temporary: + temporary_root = Path(temporary) + source_root = _extract_pinned_archive(pin, temporary_root) + output_dir = temporary_root / "wheel-output" + output_dir.mkdir() + command = [ + python, + "-m", + "pip", + "wheel", + "--no-deps", + "--wheel-dir", + str(output_dir), + str(source_root), + ] + completed = subprocess.run( # noqa: S603 + command, + capture_output=True, + check=False, + text=True, + env=_subprocess_environment(public_pip_index=True), + ) + if completed.returncode: + detail = completed.stderr.strip() or completed.stdout.strip() + raise CtpSourcePinError(f"pinned bt_api_ctp wheel build failed: {detail}") + wheels = sorted(output_dir.glob("*.whl")) + if len(wheels) != 1: + raise CtpSourcePinError(f"expected one pinned bt_api_ctp wheel, found {len(wheels)}") + built_wheel = wheels[0] + wheel_name, wheel_version = _read_wheel_identity(built_wheel) + if wheel_name != pin.package_name or wheel_version != pin.package_version: + raise CtpSourcePinError( + f"pinned CTP wheel metadata mismatch: Name={wheel_name!r}, Version={wheel_version!r}" + ) + final_wheel = wheel_dir / built_wheel.name + final_wheel.write_bytes(built_wheel.read_bytes()) + + wheel_name, wheel_version = _read_wheel_identity(final_wheel) + if wheel_name != pin.package_name or wheel_version != pin.package_version: + raise CtpSourcePinError("copied local CTP wheel metadata changed after the source build") + wheel_sha256 = _sha256(final_wheel) + install_env = _subprocess_environment() + install = subprocess.run( # noqa: S603 + [ + python, + "-m", + "pip", + "install", + "--no-deps", + "--force-reinstall", + str(final_wheel), + ], + capture_output=True, + check=False, + text=True, + env=install_env, + ) + if install.returncode: + detail = install.stderr.strip() or install.stdout.strip() + raise CtpSourcePinError(f"installing pinned bt_api_ctp wheel failed: {detail}") + + receipt = CtpWheelReceipt( + parent_commit=pin.parent_commit, + source_commit=pin.source_commit, + source_tree=pin.source_tree, + source_origin=pin.source_origin, + package_name=pin.package_name, + package_version=pin.package_version, + minimum_version=pin.minimum_version, + minimum_base_version=pin.minimum_base_version, + wheel_filename=final_wheel.name, + wheel_sha256=wheel_sha256, + wheel_path=final_wheel, + ) + if _sha256(final_wheel) != receipt.wheel_sha256: + raise CtpSourcePinError("local bt_api_ctp wheel hash changed before installation") + _verify_local_wheel_origin( + receipt.package_name, + receipt.package_version, + receipt.wheel_path, + receipt.wheel_sha256, + ) + return receipt + + +def _write_github_environment(path: Path, receipt: CtpWheelReceipt) -> None: + values = { + "BT_API_CTP_SOURCE_SHA": receipt.source_commit, + "BT_API_CTP_SOURCE_TREE_SHA": receipt.source_tree, + "BT_API_CTP_SOURCE_ORIGIN": receipt.source_origin, + "BT_API_CTP_SOURCE_VERSION": receipt.package_version, + "BT_API_CTP_WHEEL_SHA256": receipt.wheel_sha256, + "BT_API_CTP_WHEEL_PATH": str(receipt.wheel_path), + } + with path.open("a", encoding="utf-8", newline="\n") as handle: + for name, value in values.items(): + handle.write(f"{name}={_safe_env_value(value)}\n") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--repository-root", + type=Path, + default=Path(__file__).resolve().parents[2], + ) + parser.add_argument("--wheel-dir", type=Path) + parser.add_argument("--github-env", type=Path) + parser.add_argument("--verify-installed", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + repository_root = args.repository_root.resolve() + if args.verify_installed: + receipt = _verify_installed_ctp_receipt(repository_root) + _verify_base_receipt(repository_root) + else: + if args.wheel_dir is None: + raise CtpSourcePinError( + "--wheel-dir is required when building the pinned CTP wheel" + ) + receipt = build_and_install_ctp_wheel(repository_root, args.wheel_dir) + if args.github_env: + _write_github_environment(args.github_env, receipt) + except (BaseSourcePinError, CtpSourcePinError, OSError) as exc: + print(f"CTP source pin bootstrap failed: {exc}", file=sys.stderr) + return 1 + print(json.dumps(asdict(receipt), default=str, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/submodule_validation.py b/scripts/ci/submodule_validation.py index 261e9775..6f6ccad9 100644 --- a/scripts/ci/submodule_validation.py +++ b/scripts/ci/submodule_validation.py @@ -421,6 +421,9 @@ def validate_package( "-m", "pip", "wheel", + # Install/check below validates runtime dependencies against the + # exact local base wheel already installed in this environment. + "--no-deps", *pip_source_args(wheelhouse), "--wheel-dir", str(plugin_dist_dir), diff --git a/scripts/ci/verify_wheel_contract.py b/scripts/ci/verify_wheel_contract.py index f8adedc5..74e8d5d4 100644 --- a/scripts/ci/verify_wheel_contract.py +++ b/scripts/ci/verify_wheel_contract.py @@ -16,6 +16,7 @@ from typing import Any if __package__: + from .base_source_pin import BaseSourcePinError, BaseWheelReceipt, build_pinned_base_wheel from .offline_pip import ( WheelhousePathError, pip_source_args, @@ -23,6 +24,7 @@ resolve_wheelhouse_path, ) else: + from base_source_pin import BaseSourcePinError, BaseWheelReceipt, build_pinned_base_wheel from offline_pip import ( WheelhousePathError, pip_source_args, @@ -82,6 +84,30 @@ def _venv_python(venv_dir: Path) -> Path: return venv_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python") +def _require_venv_package_path(package_file: str, venv_dir: Path, package_name: str) -> str: + package_path = Path(package_file).resolve() + venv_path = venv_dir.resolve() + if package_path.is_relative_to(venv_path): + relative_parts = tuple( + part.casefold() for part in package_path.relative_to(venv_path).parts + ) + expected_package = package_name.casefold() + has_expected_package_path = any( + part == "site-packages" + and index + 1 < len(relative_parts) + and relative_parts[index + 1] == expected_package + for index, part in enumerate(relative_parts) + ) + else: + has_expected_package_path = False + if not has_expected_package_path: + raise WheelContractError( + f"installed {package_name} package probe resolved outside the virtualenv site-packages: " + f"{package_path}" + ) + return package_path.as_posix() + + def _isolated_subprocess_env(wheelhouse: Path | None = None) -> dict[str, str]: """Return an environment that cannot join a parent pytest-cov session. @@ -123,8 +149,8 @@ def _head_sha() -> str: def _isolated_install_probe( - wheel: Path, wheelhouse: Path | None = None -) -> tuple[str, dict[str, Any], dict[str, Any]]: + wheel: Path, wheelhouse: Path | None = None, *, base_wheel_dir: Path +) -> tuple[str, dict[str, Any], dict[str, Any], BaseWheelReceipt]: with tempfile.TemporaryDirectory(prefix="bt-api-py-wheel-contract-") as temp_dir: temp_root = Path(temp_dir) venv_dir = temp_root / "venv" @@ -141,6 +167,40 @@ def _isolated_install_probe( ) python = _venv_python(venv_dir) + base_wheel_dir = base_wheel_dir.resolve() + try: + base_receipt = build_pinned_base_wheel( + REPOSITORY_ROOT, + base_wheel_dir, + build_wheelhouse=wheelhouse, + ) + except (BaseSourcePinError, WheelhousePathError, OSError) as exc: + raise WheelContractError( + f"could not build the pinned bt_api_base wheel: {exc}" + ) from exc + + base_env = _isolated_subprocess_env(base_wheel_dir) + install_base = _run( + [ + str(python), + "-m", + "pip", + "install", + *pip_source_args(base_wheel_dir), + "--disable-pip-version-check", + "--no-deps", + "--force-reinstall", + str(base_receipt.wheel_path), + ], + cwd=temp_root, + env=base_env, + ) + if install_base.returncode != 0: + raise WheelContractError( + "isolated pinned base wheel installation failed: " + f"{install_base.stderr.strip() or install_base.stdout.strip()}" + ) + install = _run( [ str(python), @@ -149,10 +209,11 @@ def _isolated_install_probe( "install", *pip_source_args(wheelhouse), "--disable-pip-version-check", - "--force-reinstall", # This is an installed-package probe. Resolve the wheel's # declared runtime dependencies instead of relying on whatever # happens to be present in the runner's system site-packages. + # The pinned base wheel is already installed into this fresh + # venv and satisfies the candidate wheel's base requirement. str(wheel), ], cwd=temp_root, @@ -177,12 +238,18 @@ def _isolated_install_probe( "-c", ( "import importlib.resources as resources, json, pathlib; " + "from importlib import metadata; " "import bt_api_base, bt_api_py; " "from bt_api_py._plugin_catalog import PluginCatalog; " "resource = resources.files('bt_api_py.configs').joinpath(" "'exchange-bundles.toml'); " + "base_distribution = metadata.distribution('bt_api_base'); " + "base_direct_url = json.loads(base_distribution.read_text('direct_url.json') " + "or '{}'); " "payload = {'package_file': str(pathlib.Path(bt_api_py.__file__).resolve()), " "'base_package_file': str(pathlib.Path(bt_api_base.__file__).resolve()), " + "'base_version': base_distribution.version, " + "'base_direct_url': base_direct_url, " "'resource': str(resource), 'resource_is_file': resource.is_file(), " "'bundles': PluginCatalog().list_bundles()}; " "assert payload['resource_is_file']; " @@ -204,21 +271,34 @@ def _isolated_install_probe( f"resource probe did not return JSON: {probe.stdout!r}" ) from exc - package_file = str(probe_payload["package_file"]).replace("\\", "/") - if "site-packages/bt_api_py" not in package_file: + package_file = _require_venv_package_path( + str(probe_payload["package_file"]), venv_dir, "bt_api_py" + ) + base_package_file = _require_venv_package_path( + str(probe_payload["base_package_file"]), venv_dir, "bt_api_base" + ) + probe_payload["base_package_file"] = base_package_file + if probe_payload["base_version"] != base_receipt.package_version: raise WheelContractError( - "installed package probe resolved outside the virtualenv site-packages: " - f"{package_file}" + "installed base package version does not match the parent-pinned source wheel: " + f"{probe_payload['base_version']} != {base_receipt.package_version}" ) - base_package_path = Path(str(probe_payload["base_package_file"])).resolve() - if ( - not base_package_path.is_relative_to(venv_dir.resolve()) - or "site-packages" not in base_package_path.parts - or "bt_api_base" not in base_package_path.parts - ): + expected_wheel_url = base_receipt.wheel_path.resolve().as_uri() + recorded_wheel_url = probe_payload["base_direct_url"].get("url") + if recorded_wheel_url != expected_wheel_url: raise WheelContractError( - "installed base package probe resolved outside the virtualenv site-packages: " - f"{base_package_path}" + "installed base package PEP 610 URL does not identify the exact pinned local wheel: " + f"{recorded_wheel_url!r} != {expected_wheel_url!r}" + ) + archive_info = probe_payload["base_direct_url"].get("archive_info") or {} + recorded_hash = archive_info.get("hash") + if recorded_hash is None: + recorded_hash = (archive_info.get("hashes") or {}).get("sha256") + if recorded_hash: + recorded_hash = f"sha256={recorded_hash}" + if recorded_hash != f"sha256={base_receipt.wheel_sha256}": + raise WheelContractError( + "installed base package PEP 610 wheel hash does not match the pinned source wheel" ) doctor = _run( @@ -252,11 +332,13 @@ def _isolated_install_probe( "stdout_sha256": _sha256(doctor.stdout.encode()), "stderr_sha256": _sha256(doctor.stderr.encode()), }, + base_receipt, ) def verify(dist_dir: Path, wheelhouse: Path | None = None) -> dict[str, Any]: """Build an evidence receipt for the source, wheel, and sdist resource contract.""" + dist_dir = dist_dir.resolve() wheelhouse = resolve_wheelhouse_path(wheelhouse) source_resource = REPOSITORY_ROOT / PACKAGE_RESOURCE @@ -275,7 +357,11 @@ def verify(dist_dir: Path, wheelhouse: Path | None = None) -> dict[str, Any]: f"source, wheel, and sdist exchange-bundles.toml hashes do not match: {resource_hashes}" ) - package_file, probe, doctor = _isolated_install_probe(wheel, wheelhouse) + package_file, probe, doctor, base_receipt = _isolated_install_probe( + wheel, + wheelhouse, + base_wheel_dir=dist_dir / "bt_api_base_source", + ) return { "schema_version": 1, "generated_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), @@ -292,6 +378,19 @@ def verify(dist_dir: Path, wheelhouse: Path | None = None) -> dict[str, Any]: "resource_sha256": resource_hashes, "package_file": package_file, "base_package_file": probe["base_package_file"], + "base_source": { + "parent_commit": base_receipt.parent_commit, + "source_commit": base_receipt.source_commit, + "source_tree": base_receipt.source_tree, + "source_origin": base_receipt.source_origin, + "package_name": base_receipt.package_name, + "package_version": base_receipt.package_version, + "minimum_version": base_receipt.minimum_version, + "wheel_filename": base_receipt.wheel_filename, + "wheel_path": base_receipt.wheel_path.relative_to(dist_dir).as_posix(), + "wheel_path_url": base_receipt.wheel_path.resolve().as_uri(), + "wheel_sha256": base_receipt.wheel_sha256, + }, "probe": probe, "doctor": doctor, } diff --git a/tests/offline_wheelhouse.py b/tests/offline_wheelhouse.py index d5892a7c..eae8a3e4 100644 --- a/tests/offline_wheelhouse.py +++ b/tests/offline_wheelhouse.py @@ -2,10 +2,8 @@ from __future__ import annotations -import os import shutil import subprocess -import sys import sysconfig import tempfile import tomllib @@ -17,10 +15,9 @@ from packaging.version import Version from wheel.wheelfile import WheelFile -from scripts.ci.offline_pip import pip_source_args, pip_source_environment +from scripts.ci.base_source_pin import verify_base_source_pin REPOSITORY_ROOT = Path(__file__).resolve().parents[1] -BASE_SOURCE = REPOSITORY_ROOT / "bt_api" / "bt_api_base" PYTEST_SOCKET_VERSION = "0.7.0" @@ -252,58 +249,34 @@ def build_validator_wheelhouse(destination: Path) -> Path: return wheelhouse -def _build_local_base_wheel(wheelhouse: Path) -> None: - if not (BASE_SOURCE / "pyproject.toml").is_file(): - raise RuntimeError(f"local bt_api_base source is unavailable: {BASE_SOURCE}") - - with tempfile.TemporaryDirectory(prefix="bt-api-base-wheel-source-") as temp_dir: - source = Path(temp_dir) / "bt_api_base" - shutil.copytree( - BASE_SOURCE, - source, - ignore=shutil.ignore_patterns( - ".git", "build", "dist", "*.egg-info", "__pycache__", ".pytest_cache" - ), - ) - environment = pip_source_environment(os.environ, wheelhouse) - command = [ - sys.executable, - "-m", - "pip", - "wheel", - *pip_source_args(wheelhouse), - "--wheel-dir", - str(wheelhouse), - str(source), - ] - result = subprocess.run( - command, - cwd=temp_dir, - env=environment, - capture_output=True, - check=False, - text=True, - ) - if result.returncode != 0: - raise RuntimeError( - "local bt_api_base wheel build failed: " - f"{result.stderr.strip() or result.stdout.strip()}" - ) - if not list(wheelhouse.glob("bt_api_base-0.15.4-*.whl")): - raise RuntimeError("local bt_api_base build did not produce version 0.15.4") - - def build_project_wheelhouse(destination: Path) -> Path: - """Build a temporary root/base runtime dependency wheelhouse from local installs.""" + """Build a temporary root/base dependency wheelhouse from local installs.""" wheelhouse = destination.resolve() wheelhouse.mkdir(parents=True, exist_ok=True) root_pyproject = REPOSITORY_ROOT / "pyproject.toml" - base_pyproject = BASE_SOURCE / "pyproject.toml" + base_pin = verify_base_source_pin(REPOSITORY_ROOT) + base_metadata = subprocess.run( + [ # noqa: S607 - Git is required for the checked-out source pin. + "git", + "-C", + str(base_pin.source_path), + "show", + f"{base_pin.source_commit}:pyproject.toml", + ], + capture_output=True, + check=False, + text=True, + ) + if base_metadata.returncode != 0: + raise RuntimeError("pinned bt_api_base source metadata could not be read") + pinned_base_config = tomllib.loads(base_metadata.stdout) root_requirements = _project_requirements(root_pyproject, "dependencies") - base_requirements = _project_requirements(base_pyproject, "dependencies") build_requirements = [ *_build_requirements(root_pyproject), - *_build_requirements(base_pyproject), + *[str(item) for item in pinned_base_config.get("build-system", {}).get("requires", [])], + ] + base_requirements = [ + str(item) for item in pinned_base_config.get("project", {}).get("dependencies", []) ] root_requirements = [ @@ -313,5 +286,4 @@ def build_project_wheelhouse(destination: Path) -> Path: ] requirements = [*root_requirements, *base_requirements, *build_requirements] _repackage_dependency_closure(wheelhouse, requirements, excluded={"bt_api_base"}) - _build_local_base_wheel(wheelhouse) return wheelhouse diff --git a/tests/scripts/test_base_source_pin.py b/tests/scripts/test_base_source_pin.py new file mode 100644 index 00000000..8985d856 --- /dev/null +++ b/tests/scripts/test_base_source_pin.py @@ -0,0 +1,274 @@ +"""Offline tests for binding a base wheel to the parent repository source pin.""" + +from __future__ import annotations + +import json +import os +import subprocess +import zipfile +from pathlib import Path + +import pytest + +import scripts.ci.base_source_pin as base_source_pin +from scripts.ci.base_source_pin import ( + BASE_ORIGIN, + BaseSourcePinError, + build_pinned_base_wheel, + verify_base_source_pin, +) + + +def _git(repository: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", "-C", str(repository), *arguments], # noqa: S607 + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _commit(repository: Path, message: str) -> None: + _git(repository, "add", "-A") + environment = { + "GIT_AUTHOR_NAME": "CI source pin test", + "GIT_AUTHOR_EMAIL": "ci-source-pin@example.invalid", + "GIT_COMMITTER_NAME": "CI source pin test", + "GIT_COMMITTER_EMAIL": "ci-source-pin@example.invalid", + } + subprocess.run( + ["git", "-C", str(repository), "commit", "-m", message], # noqa: S607 + check=True, + capture_output=True, + env={**os.environ, **environment}, + ) + + +def _make_repositories( + root: Path, + *, + source_name: str = "bt_api_base", + source_version: str = "0.15.5", + minimum_version: str = "0.15.5", +) -> tuple[Path, Path, str]: + parent = root / "parent" + source = parent / "bt_api" / "bt_api_base" + source.mkdir(parents=True) + _git(parent, "init", "-q") + _git(source, "init", "-q") + _git(source, "config", "user.name", "CI source pin test") + _git(source, "config", "user.email", "ci-source-pin@example.invalid") + _git(source, "remote", "add", "origin", BASE_ORIGIN) + + (source / "pyproject.toml").write_text( + f'[project]\nname = "{source_name}"\nversion = "{source_version}"\n', + encoding="utf-8", + ) + _commit(source, "source") + source_sha = _git(source, "rev-parse", "HEAD") + + (parent / ".gitmodules").write_text( + f'[submodule "bt_api/bt_api_base"]\n\tpath = bt_api/bt_api_base\n\turl = {BASE_ORIGIN}\n', + encoding="utf-8", + ) + (parent / "pyproject.toml").write_text( + "[project]\n" + "name = 'parent-test'\n" + "version = '0.0.0'\n" + f"dependencies = ['bt_api_base>={minimum_version}']\n", + encoding="utf-8", + ) + _git(parent, "add", ".gitmodules", "pyproject.toml") + _git( + parent, + "update-index", + "--add", + "--cacheinfo", + f"160000,{source_sha},bt_api/bt_api_base", + ) + _commit(parent, "parent") + return parent, source, source_sha + + +def test_source_pin_uses_parent_gitlink_and_checks_origin_and_package_metadata( + tmp_path: Path, +) -> None: + parent, source, source_sha = _make_repositories(tmp_path) + + pin = verify_base_source_pin(parent) + + assert pin.source_commit == source_sha + assert pin.source_tree == _git(source, "rev-parse", "HEAD^{tree}") + assert pin.source_origin == BASE_ORIGIN + assert pin.package_name == "bt_api_base" + assert pin.package_version == "0.15.5" + assert pin.minimum_version == "0.15.5" + + +def test_source_pin_rejects_a_noncanonical_gitmodule_origin(tmp_path: Path) -> None: + parent, _, _ = _make_repositories(tmp_path) + gitmodules = parent / ".gitmodules" + gitmodules.write_text( + gitmodules.read_text(encoding="utf-8").replace( + BASE_ORIGIN, "https://example.invalid/base.git" + ), + encoding="utf-8", + ) + _git(parent, "add", ".gitmodules") + _commit(parent, "alter origin") + + with pytest.raises(BaseSourcePinError, match="origin mismatch"): + verify_base_source_pin(parent) + + +def test_source_pin_does_not_log_credentials_from_a_remote_url(tmp_path: Path) -> None: + parent, source, _ = _make_repositories(tmp_path) + _git(source, "remote", "set-url", "origin", "https://user:fake-secret@example.invalid/base.git") + + with pytest.raises(BaseSourcePinError, match="origin mismatch") as error: + verify_base_source_pin(parent) + + assert "fake-secret" not in str(error.value) + + +def test_pip_child_processes_receive_only_allowlisted_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + parent, _, _ = _make_repositories(tmp_path) + monkeypatch.setenv("CODECOV_TOKEN", "fake-codecov-secret") + monkeypatch.setenv("GITHUB_TOKEN", "fake-github-secret") + monkeypatch.setenv("PIP_EXTRA_INDEX_URL", "https://user:fake-pip-secret@example.invalid/simple") + monkeypatch.setenv("PYTHONPATH", "fake-private-pythonpath") + + original_run = subprocess.run + pip_child_environments: list[dict[str, str]] = [] + + def capture_run( + command: list[str], *args: object, **kwargs: object + ) -> subprocess.CompletedProcess: + if len(command) >= 3 and command[1:3] == ["-m", "pip"]: + environment = kwargs["env"] + assert isinstance(environment, dict) + pip_child_environments.append(environment) + if "wheel" in command: + output_dir = Path(command[command.index("--wheel-dir") + 1]) + wheel_path = output_dir / "bt_api_base-0.15.5-py3-none-any.whl" + with zipfile.ZipFile(wheel_path, "w") as wheel: + wheel.writestr( + "bt_api_base-0.15.5.dist-info/METADATA", + "Name: bt_api_base\nVersion: 0.15.5\n", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + return original_run(command, *args, **kwargs) + + monkeypatch.setattr(base_source_pin.subprocess, "run", capture_run) + monkeypatch.setattr(base_source_pin, "_verify_installed_wheel", lambda _receipt: None) + + base_source_pin.build_and_install_base_wheel(parent, tmp_path / "wheelhouse") + + assert len(pip_child_environments) == 2 + for environment in pip_child_environments: + allowed_keys = set(base_source_pin.SUBPROCESS_ENV_ALLOWLIST) | { + "PIP_CONFIG_FILE", + "PIP_INDEX_URL", + } + assert set(environment) <= allowed_keys + assert "CODECOV_TOKEN" not in environment + assert "GITHUB_TOKEN" not in environment + assert "PIP_EXTRA_INDEX_URL" not in environment + assert "PYTHONPATH" not in environment + for name in base_source_pin.SUBPROCESS_ENV_ALLOWLIST: + if name in os.environ: + assert environment[name] == os.environ[name] + assert environment["PIP_CONFIG_FILE"] == os.devnull + assert "fake-codecov-secret" not in json.dumps(environment) + assert "fake-github-secret" not in json.dumps(environment) + assert "fake-pip-secret" not in json.dumps(environment) + assert pip_child_environments[0]["PIP_INDEX_URL"] == "https://pypi.org/simple/" + assert "PIP_INDEX_URL" not in pip_child_environments[1] + + +def test_pinned_source_wheel_can_be_built_from_a_local_wheelhouse_without_installing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + parent, _, source_sha = _make_repositories(tmp_path) + build_wheelhouse = tmp_path / "build-wheelhouse" + build_wheelhouse.mkdir() + original_run = subprocess.run + pip_calls: list[tuple[list[str], dict[str, str]]] = [] + + def capture_run( + command: list[str], *args: object, **kwargs: object + ) -> subprocess.CompletedProcess: + if len(command) >= 3 and command[1:3] == ["-m", "pip"]: + environment = kwargs["env"] + assert isinstance(environment, dict) + pip_calls.append((command, environment)) + assert "wheel" in command + output_dir = Path(command[command.index("--wheel-dir") + 1]) + wheel_path = output_dir / "bt_api_base-0.15.5-py3-none-any.whl" + with zipfile.ZipFile(wheel_path, "w") as wheel: + wheel.writestr( + "bt_api_base-0.15.5.dist-info/METADATA", + "Name: bt_api_base\nVersion: 0.15.5\n", + ) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + return original_run(command, *args, **kwargs) + + monkeypatch.setattr(base_source_pin.subprocess, "run", capture_run) + + receipt = build_pinned_base_wheel( + parent, + tmp_path / "wheelhouse", + build_wheelhouse=build_wheelhouse, + ) + + assert len(pip_calls) == 1 + command, environment = pip_calls[0] + assert "--no-index" in command + assert command[command.index("--find-links") + 1] == str(build_wheelhouse.resolve()) + assert "PIP_INDEX_URL" not in environment + assert "PIP_EXTRA_INDEX_URL" not in environment + assert receipt.source_commit == source_sha + assert receipt.source_origin == BASE_ORIGIN + assert receipt.package_version == "0.15.5" + assert receipt.wheel_path.is_file() + assert receipt.wheel_sha256 == base_source_pin._sha256(receipt.wheel_path) + + +def test_source_pin_rejects_a_checkout_that_does_not_match_the_gitlink( + tmp_path: Path, +) -> None: + parent, source, source_sha = _make_repositories(tmp_path) + (source / "README.md").write_text("different checkout\n", encoding="utf-8") + _commit(source, "advance source") + + with pytest.raises(BaseSourcePinError, match=f"does not match parent gitlink {source_sha}"): + verify_base_source_pin(parent) + + +@pytest.mark.parametrize( + ("source_name", "source_version", "minimum_version", "expected"), + [ + ("unrelated_package", "0.15.5", "0.15.5", "package name"), + ("bt_api_base", "0.15.4", "0.15.5", "does not satisfy parent"), + ("bt_api_base", "0.15.5", "0.15.4", "lower bound must remain >=0.15.5"), + ], +) +def test_source_pin_rejects_wrong_package_identity_or_version_floor( + tmp_path: Path, + source_name: str, + source_version: str, + minimum_version: str, + expected: str, +) -> None: + parent, _, _ = _make_repositories( + tmp_path, + source_name=source_name, + source_version=source_version, + minimum_version=minimum_version, + ) + + with pytest.raises(BaseSourcePinError, match=expected): + verify_base_source_pin(parent) diff --git a/tests/scripts/test_check_format_ratchet.py b/tests/scripts/test_check_format_ratchet.py index c51683d2..73eb209e 100644 --- a/tests/scripts/test_check_format_ratchet.py +++ b/tests/scripts/test_check_format_ratchet.py @@ -94,6 +94,36 @@ def fake_ruff(*args): assert ratchet.scan_module("bt_api_example") == 0 assert calls == [("format", "--check", "bt_api/bt_api_example")] + def test_successful_no_python_warning_is_zero_debt(self, monkeypatch): + def fake_ruff(*args): + return subprocess.CompletedProcess( + args, + 0, + "", + "warning: No Python files found under the given path(s)\n", + ) + + monkeypatch.setattr(ratchet, "_ruff", fake_ruff) + + assert ratchet.scan_module("bt_api_execution") == 0 + + @pytest.mark.parametrize( + ("returncode", "stdout", "stderr"), + [ + (0, "", "warning: another condition\n"), + (0, "", "warning: No Python files found under the given path(s)\nextra\n"), + (1, "", "warning: No Python files found under the given path(s)\n"), + ], + ) + def test_other_empty_output_still_fails_closed(self, monkeypatch, returncode, stdout, stderr): + def fake_ruff(*args): + return subprocess.CompletedProcess(args, returncode, stdout, stderr) + + monkeypatch.setattr(ratchet, "_ruff", fake_ruff) + + with pytest.raises(ratchet.FormatScanError): + ratchet.scan_module("bt_api_execution") + def test_ruff_process_uses_fixed_root_and_never_a_shell(self, monkeypatch): calls = [] completed = subprocess.CompletedProcess(["ruff", "--version"], 0, "ruff 0.16.2\n", "") @@ -148,14 +178,18 @@ def test_committed_baseline_has_schema_scope_and_all_module_counts(self): "bt_api_coinbase": 23, "bt_api_ctp": 25, "bt_api_dydx": 20, + "bt_api_execution": 0, "bt_api_gateio": 25, + "bt_api_gateway": 0, "bt_api_htx": 23, "bt_api_hyperliquid": 23, "bt_api_ib_web": 40, "bt_api_kraken": 7, "bt_api_mexc": 12, + "bt_api_monitor": 0, "bt_api_mt5": 7, "bt_api_okx": 85, + "bt_api_risk": 0, } assert payload["schema_version"] == ratchet.SCHEMA_VERSION diff --git a/tests/scripts/test_ctp_source_pin.py b/tests/scripts/test_ctp_source_pin.py new file mode 100644 index 00000000..ad9fe4fc --- /dev/null +++ b/tests/scripts/test_ctp_source_pin.py @@ -0,0 +1,330 @@ +"""Offline tests for binding the CTP CI wheel to its exact parent gitlink.""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import zipfile +from pathlib import Path +from typing import Any + +import pytest + +import scripts.ci.ctp_source_pin as ctp_source_pin +from scripts.ci.base_source_pin import SUBPROCESS_ENV_ALLOWLIST +from scripts.ci.ctp_source_pin import ( + CTP_ORIGIN, + CtpSourcePinError, + _extract_pinned_archive, + build_and_install_ctp_wheel, + verify_ctp_source_pin, +) + +BASE_ORIGIN = "https://github.com/cloudQuant/bt_api_base.git" + + +def _git(repository: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", "-C", str(repository), *arguments], # noqa: S607 + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _commit(repository: Path, message: str) -> None: + _git(repository, "add", "-A") + environment = { + "GIT_AUTHOR_NAME": "CI source pin test", + "GIT_AUTHOR_EMAIL": "ci-source-pin@example.invalid", + "GIT_COMMITTER_NAME": "CI source pin test", + "GIT_COMMITTER_EMAIL": "ci-source-pin@example.invalid", + } + subprocess.run( + ["git", "-C", str(repository), "commit", "-m", message], # noqa: S607 + check=True, + capture_output=True, + env={**os.environ, **environment}, + ) + + +def _make_source(path: Path, origin: str, pyproject: str, *, extra_file: str = "") -> str: + path.mkdir(parents=True) + _git(path, "init", "-q") + _git(path, "config", "user.name", "CI source pin test") + _git(path, "config", "user.email", "ci-source-pin@example.invalid") + _git(path, "remote", "add", "origin", origin) + (path / "pyproject.toml").write_text(pyproject, encoding="utf-8") + if extra_file: + (path / "README.md").write_text(extra_file, encoding="utf-8") + _commit(path, "source") + return _git(path, "rev-parse", "HEAD") + + +def _make_repositories( + root: Path, + *, + ctp_version: str = "2.0.3", + base_version: str = "0.15.5", + ctp_requirement: str = "bt_api_ctp>=2.0.3,<3.0", + ctp_origin: str = CTP_ORIGIN, +) -> tuple[Path, Path, Path, str, str]: + parent = root / "parent" + base = parent / "bt_api" / "bt_api_base" + ctp = parent / "bt_api" / "bt_api_ctp" + base_sha = _make_source( + base, + BASE_ORIGIN, + f'[project]\nname = "bt_api_base"\nversion = "{base_version}"\n', + ) + ctp_sha = _make_source( + ctp, + ctp_origin, + "[build-system]\nrequires = ['setuptools>=69', 'wheel']\n" + "build-backend = 'setuptools.build_meta'\n" + "[project]\nname = 'bt_api_ctp'\n" + f"version = '{ctp_version}'\n" + "requires-python = '>=3.9'\n" + "dependencies = ['bt_api_base>=0.15.5,<1.0']\n", + extra_file="pinned source snapshot\n", + ) + (ctp / "setup.py").write_text("# native extension build marker\n", encoding="utf-8") + _commit(ctp, "add setup marker") + ctp_sha = _git(ctp, "rev-parse", "HEAD") + + parent.mkdir(exist_ok=True) + _git(parent, "init", "-q") + _git(parent, "config", "user.name", "CI source pin test") + _git(parent, "config", "user.email", "ci-source-pin@example.invalid") + (parent / ".gitmodules").write_text( + f'[submodule "bt_api/bt_api_base"]\n\tpath = bt_api/bt_api_base\n\turl = {BASE_ORIGIN}\n' + f'[submodule "bt_api/bt_api_ctp"]\n\tpath = bt_api/bt_api_ctp\n\turl = {ctp_origin}\n', + encoding="utf-8", + ) + (parent / "pyproject.toml").write_text( + "[project]\nname = 'parent-test'\nversion = '0.0.0'\n" + f"dependencies = ['bt_api_base>={base_version}']\n" + "[project.optional-dependencies]\n" + f"core-reference = ['{ctp_requirement}']\n", + encoding="utf-8", + ) + bundle = parent / "bt_api_py" / "configs" / "exchange-bundles.toml" + bundle.parent.mkdir(parents=True) + bundle.write_text( + "[bundles.core-reference]\n" + "description = 'test'\n" + "[[bundles.core-reference.venues]]\n" + "package = 'bt_api_ctp'\n" + "plugin = 'ctp'\n" + "exchange = 'CTP___FUTURE'\n" + "min_version = '2.0.3'\n", + encoding="utf-8", + ) + _git( + parent, + "add", + ".gitmodules", + "pyproject.toml", + "bt_api_py/configs/exchange-bundles.toml", + ) + _git( + parent, + "update-index", + "--add", + "--cacheinfo", + f"160000,{base_sha},bt_api/bt_api_base", + ) + _git( + parent, + "update-index", + "--add", + "--cacheinfo", + f"160000,{ctp_sha},bt_api/bt_api_ctp", + ) + _commit(parent, "parent") + return parent, base, ctp, base_sha, ctp_sha + + +def test_ctp_pin_reads_gitlink_and_validates_bundle_and_runtime_floors( + tmp_path: Path, +) -> None: + parent, _, ctp, _, ctp_sha = _make_repositories(tmp_path) + + pin = verify_ctp_source_pin(parent) + + assert pin.source_commit == ctp_sha + assert pin.source_tree == _git(ctp, "rev-parse", "HEAD^{tree}") + assert pin.source_origin == CTP_ORIGIN + assert pin.package_name == "bt_api_ctp" + assert pin.package_version == "2.0.3" + assert pin.minimum_version == "2.0.3" + assert pin.minimum_base_version == "0.15.5" + + +def test_ctp_archive_is_built_from_gitlink_and_ignores_worktree_edits( + tmp_path: Path, +) -> None: + parent, _, ctp, _, _ = _make_repositories(tmp_path) + pin = verify_ctp_source_pin(parent) + (ctp / "README.md").write_text("uncommitted change\n", encoding="utf-8") + (ctp / "untracked.txt").write_text("not in gitlink\n", encoding="utf-8") + + archive_root = _extract_pinned_archive(pin, tmp_path / "extracted") + + assert (archive_root / "README.md").read_text(encoding="utf-8") == "pinned source snapshot\n" + assert not (archive_root / "untracked.txt").exists() + + +@pytest.mark.parametrize( + ("ctp_version", "ctp_requirement", "expected"), + [ + ("2.0.2", "bt_api_ctp>=2.0.3,<3.0", "must be >=2.0.3"), + ("2.0.3", "bt_api_ctp>=2.0.2,<3.0", "lower bound must remain >=2.0.3"), + ], +) +def test_ctp_pin_rejects_source_or_parent_version_below_floor( + tmp_path: Path, ctp_version: str, ctp_requirement: str, expected: str +) -> None: + parent, *_ = _make_repositories( + tmp_path, ctp_version=ctp_version, ctp_requirement=ctp_requirement + ) + + with pytest.raises(CtpSourcePinError, match=expected): + verify_ctp_source_pin(parent) + + +def test_ctp_pin_rejects_a_noncanonical_source_origin(tmp_path: Path) -> None: + parent, *_ = _make_repositories(tmp_path, ctp_origin="https://example.invalid/ctp.git") + + with pytest.raises(CtpSourcePinError, match="origin mismatch"): + verify_ctp_source_pin(parent) + + +def test_build_install_receipt_uses_allowlisted_env_and_pep610_wheel_hash( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + parent, base, _, base_sha, _ = _make_repositories(tmp_path) + base_tree = _git(base, "rev-parse", "HEAD^{tree}") + fake_base_wheel = tmp_path / "base-wheelhouse" / "bt_api_base-0.15.5-py3-none-any.whl" + fake_base_wheel.parent.mkdir() + fake_base_wheel.write_bytes(b"verified base wheel placeholder") + base_sha256 = hashlib.sha256(fake_base_wheel.read_bytes()).hexdigest() + monkeypatch.setenv("BT_API_BASE_SOURCE_SHA", base_sha) + monkeypatch.setenv("BT_API_BASE_SOURCE_TREE_SHA", base_tree) + monkeypatch.setenv("BT_API_BASE_SOURCE_ORIGIN", BASE_ORIGIN) + monkeypatch.setenv("BT_API_BASE_SOURCE_VERSION", "0.15.5") + monkeypatch.setenv("BT_API_BASE_WHEEL_PATH", str(fake_base_wheel)) + monkeypatch.setenv("BT_API_BASE_WHEEL_SHA256", base_sha256) + monkeypatch.setenv("CODECOV_TOKEN", "fake-codecov-secret") + monkeypatch.setenv("GITHUB_TOKEN", "fake-github-secret") + monkeypatch.setenv("PIP_EXTRA_INDEX_URL", "https://user:fake-pip-secret@example.invalid/simple") + monkeypatch.setenv("PYTHONPATH", "fake-private-pythonpath") + + direct_urls: dict[str, dict[str, Any]] = { + "bt_api_base": { + "url": fake_base_wheel.resolve().as_uri(), + "archive_info": {"hashes": {"sha256": base_sha256}}, + } + } + + class FakeDistribution: + def __init__(self, package: str) -> None: + self.version = "0.15.5" if package == "bt_api_base" else "2.0.3" + self._package = package + + def read_text(self, filename: str) -> str | None: + assert filename == "direct_url.json" + return json.dumps(direct_urls.get(self._package, {})) + + from importlib import metadata + + monkeypatch.setattr(metadata, "distribution", lambda package: FakeDistribution(package)) + + original_run = subprocess.run + pip_child_environments: list[dict[str, str]] = [] + built_ctp_wheel: Path | None = None + + def fake_run( + command: list[str], *args: object, **kwargs: object + ) -> subprocess.CompletedProcess: + nonlocal built_ctp_wheel + if len(command) >= 4 and command[1:3] == ["-m", "pip"]: + environment = kwargs.get("env") + assert isinstance(environment, dict) + pip_child_environments.append(environment) + if command[3] == "wheel": + assert "--no-deps" in command + source_root = Path(command[-1]).resolve() + assert source_root.parent != (parent / "bt_api" / "bt_api_ctp").resolve() + output_dir = Path(command[command.index("--wheel-dir") + 1]) + built_ctp_wheel = output_dir / "bt_api_ctp-2.0.3-py3-none-any.whl" + with zipfile.ZipFile(built_ctp_wheel, "w") as wheel: + wheel.writestr( + "bt_api_ctp-2.0.3.dist-info/METADATA", + "Name: bt_api_ctp\nVersion: 2.0.3\n" + "Requires-Dist: bt_api_base<1.0,>=0.15.5\n", + ) + elif command[3] == "install": + assert "--no-deps" in command + installed_wheel = Path(command[-1]).resolve() + direct_urls["bt_api_ctp"] = { + "url": installed_wheel.as_uri(), + "archive_info": { + "hashes": { + "sha256": hashlib.sha256(installed_wheel.read_bytes()).hexdigest() + } + }, + } + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + return original_run(command, *args, **kwargs) + + monkeypatch.setattr(ctp_source_pin.subprocess, "run", fake_run) + receipt = build_and_install_ctp_wheel(parent, tmp_path / "ctp-wheelhouse") + + assert built_ctp_wheel is not None + assert receipt.package_name == "bt_api_ctp" + assert receipt.package_version == "2.0.3" + assert receipt.source_commit == verify_ctp_source_pin(parent).source_commit + assert receipt.wheel_sha256 == hashlib.sha256(receipt.wheel_path.read_bytes()).hexdigest() + for name, value in { + "BT_API_CTP_SOURCE_SHA": receipt.source_commit, + "BT_API_CTP_SOURCE_TREE_SHA": receipt.source_tree, + "BT_API_CTP_SOURCE_ORIGIN": receipt.source_origin, + "BT_API_CTP_SOURCE_VERSION": receipt.package_version, + "BT_API_CTP_WHEEL_PATH": str(receipt.wheel_path), + "BT_API_CTP_WHEEL_SHA256": receipt.wheel_sha256, + }.items(): + monkeypatch.setenv(name, value) + assert ctp_source_pin._verify_installed_ctp_receipt(parent) == receipt + + direct_urls["bt_api_ctp"]["url"] = ( + "https://files.pythonhosted.org/packages/bt_api_ctp-2.0.3-py3-none-any.whl" + ) + with pytest.raises(CtpSourcePinError, match="no local file PEP 610 origin"): + ctp_source_pin._verify_installed_ctp_receipt(parent) + + direct_urls["bt_api_ctp"]["url"] = receipt.wheel_path.resolve().as_uri() + original_wheel_bytes = receipt.wheel_path.read_bytes() + receipt.wheel_path.write_bytes(original_wheel_bytes + b"changed after install") + with pytest.raises(CtpSourcePinError, match="wheel changed after source-pin installation"): + ctp_source_pin._verify_installed_ctp_receipt(parent) + receipt.wheel_path.write_bytes(original_wheel_bytes) + assert ctp_source_pin._verify_installed_ctp_receipt(parent) == receipt + + assert len(pip_child_environments) == 2 + for environment in pip_child_environments: + allowed = set(SUBPROCESS_ENV_ALLOWLIST) | {"PIP_CONFIG_FILE", "PIP_INDEX_URL"} + assert set(environment) <= allowed + assert "CODECOV_TOKEN" not in environment + assert "GITHUB_TOKEN" not in environment + assert "PIP_EXTRA_INDEX_URL" not in environment + assert "PYTHONPATH" not in environment + assert "fake-codecov-secret" not in json.dumps(environment) + assert "fake-github-secret" not in json.dumps(environment) + assert "fake-pip-secret" not in json.dumps(environment) + assert environment["PIP_CONFIG_FILE"] == os.devnull + assert pip_child_environments[0]["PIP_INDEX_URL"] == "https://pypi.org/simple/" + assert "PIP_INDEX_URL" not in pip_child_environments[1] diff --git a/tests/scripts/test_install_bt_api_submodules.py b/tests/scripts/test_install_bt_api_submodules.py index 5e655337..46c91aea 100644 --- a/tests/scripts/test_install_bt_api_submodules.py +++ b/tests/scripts/test_install_bt_api_submodules.py @@ -2,8 +2,8 @@ from __future__ import annotations +from argparse import Namespace from pathlib import Path -from types import SimpleNamespace from scripts import install_bt_api_submodules as installer @@ -71,17 +71,26 @@ def test_not_packaged_submodule_is_initialized_but_never_installed(monkeypatch, ] calls = [] - args = SimpleNamespace(strategy="source-first", upgrade=True, python="/unused/python") + args = Namespace(strategy="source-first", upgrade=True, python="/unused/python") monkeypatch.setattr(installer, "installed_version", lambda _name: calls.append("version")) + + def fail_source(*_args, **_kwargs): + calls.append("source") + return False + + def fail_pypi(*_args, **_kwargs): + calls.append("pypi") + return False + monkeypatch.setattr( installer, "pip_install_source", - lambda *_args, **_kwargs: calls.append("source") or False, + fail_source, ) monkeypatch.setattr( installer, "pip_install_pypi", - lambda *_args, **_kwargs: calls.append("pypi") or False, + fail_pypi, ) result = installer.install_one(spec, args) @@ -98,7 +107,7 @@ def test_packaged_submodule_keeps_source_then_pypi_fallback(monkeypatch, tmp_pat tmp_path / "missing", "unused", ) - args = SimpleNamespace( + args = Namespace( strategy="source-first", upgrade=True, python="/unused/python", @@ -106,12 +115,17 @@ def test_packaged_submodule_keeps_source_then_pypi_fallback(monkeypatch, tmp_pat dry_run=False, ) calls = [] - monkeypatch.setattr( - installer, "pip_install_source", lambda *_args, **_kwargs: calls.append("source") or False - ) - monkeypatch.setattr( - installer, "pip_install_pypi", lambda *_args, **_kwargs: calls.append("pypi") or True - ) + + def fail_source(*_args, **_kwargs): + calls.append("source") + return False + + def succeed_pypi(*_args, **_kwargs): + calls.append("pypi") + return True + + monkeypatch.setattr(installer, "pip_install_source", fail_source) + monkeypatch.setattr(installer, "pip_install_pypi", succeed_pypi) result = installer.install_one(spec, args) @@ -130,7 +144,7 @@ def test_main_reports_not_packaged_by_default_and_strict_mode_fails(monkeypatch, summaries = [] def run(strict): - args = SimpleNamespace( + args = Namespace( packages=["execution"], strategy="source-first", with_root=False, @@ -175,7 +189,7 @@ def test_main_keeps_base_root_and_remaining_package_order_offline(monkeypatch): installer.PackageSpec("bt_api_base", "bt-api-base", Path("base"), "unused"), installer.PackageSpec("bt_api_okx", "bt-api-okx", Path("okx"), "unused"), ] - args = SimpleNamespace( + args = Namespace( packages=[], strategy="source-first", with_root=True, diff --git a/tests/scripts/test_submodule_validation.py b/tests/scripts/test_submodule_validation.py index 8ffede9a..27107d35 100644 --- a/tests/scripts/test_submodule_validation.py +++ b/tests/scripts/test_submodule_validation.py @@ -6,10 +6,15 @@ import os import subprocess import sys +from email.parser import Parser from pathlib import Path from types import MappingProxyType +from zipfile import ZipFile import pytest +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version from scripts.ci import submodule_validation from scripts.ci.offline_pip import WheelhousePathError, pip_source_environment @@ -147,6 +152,7 @@ def _write_package( root: Path, name: str, *, + version: str = "0.0.1", importable: bool = True, with_tests: bool = False, dependencies: tuple[str, ...] = (), @@ -168,7 +174,7 @@ def _write_package( [project] name = \"{name}\" -version = \"0.0.1\" +version = \"{version}\" {dependency_block}""", encoding="utf-8", ) @@ -333,6 +339,62 @@ def test_validation_installs_declared_base_wheel_dependencies(tmp_path: Path) -> assert results["bt_api_good"]["status"] == "passed" +def test_plugin_wheel_uses_the_exact_preinstalled_base_wheel(tmp_path: Path) -> None: + wheelhouse = build_validator_wheelhouse(tmp_path / "wheelhouse") + _write_package(tmp_path, "bt_api_base", version="0.15.5") + _write_package( + tmp_path, + "bt_api_ctp", + dependencies=("bt_api_base>=0.15.5,<1.0",), + ) + config = tmp_path / "configs" / "submodule-validation.toml" + config.parent.mkdir(parents=True) + config.write_text( + '[profiles.core-reference]\npackages = ["bt_api_ctp"]\ninclude_base = true\n', + encoding="utf-8", + ) + bundle_catalog = tmp_path / "bt_api_py" / "configs" / "exchange-bundles.toml" + bundle_catalog.parent.mkdir(parents=True) + bundle_catalog.write_text("[bundles]\n", encoding="utf-8") + assert not list(wheelhouse.glob("bt_api_base-*.whl")) + + artifacts = tmp_path / "artifacts" + payload = run_validation( + profile="core-reference", + repository_root=tmp_path, + artifacts_dir=artifacts, + config_path=config, + wheelhouse=wheelhouse, + ) + + results = {item["package"]: item for item in payload["packages"]} + ctp_result = results["bt_api_ctp"] + assert ctp_result["status"] == "passed" + assert ctp_result["phases"]["build"]["status"] == "passed" + assert ctp_result["phases"]["install"]["status"] == "passed" + assert ctp_result["phases"]["dependency_check"]["status"] == "passed" + assert results["bt_api_base"]["status"] == "passed" + + base_wheel = Path(ctp_result["environment"]["base_wheel"]) + plugin_wheel = Path(ctp_result["environment"]["plugin_wheel"]) + assert base_wheel.name.startswith("bt_api_base-0.15.5-") + assert base_wheel.is_file() + assert base_wheel.parent != wheelhouse + + with ZipFile(plugin_wheel) as wheel: + metadata_name = next( + name for name in wheel.namelist() if name.endswith(".dist-info/METADATA") + ) + metadata = Parser().parsestr(wheel.read(metadata_name).decode("utf-8")) + requirements = [ + Requirement(value) + for value in metadata.get_all("Requires-Dist", []) + if canonicalize_name(Requirement(value).name) == "bt-api-base" + ] + assert len(requirements) == 1 + assert Version("0.15.5") in requirements[0].specifier + + def test_dependency_check_failure_stops_before_import_and_records_evidence( monkeypatch, tmp_path: Path ) -> None: diff --git a/tests/test_package_resources.py b/tests/test_package_resources.py index f0fe3168..9783ccbc 100644 --- a/tests/test_package_resources.py +++ b/tests/test_package_resources.py @@ -15,8 +15,13 @@ import yaml from bt_api_py._plugin_catalog import PluginCatalog +from scripts.ci.base_source_pin import BASE_ORIGIN from scripts.ci.offline_pip import WheelhousePathError, pip_source_environment -from scripts.ci.verify_wheel_contract import _isolated_subprocess_env +from scripts.ci.verify_wheel_contract import ( + WheelContractError, + _isolated_subprocess_env, + _require_venv_package_path, +) from scripts.ci.verify_wheel_contract import verify as verify_wheel_contract from tests.offline_wheelhouse import build_project_wheelhouse @@ -80,6 +85,23 @@ def test_build_frontend_and_pep517_toolchain_are_declared() -> None: assert "numpy>=1.26.0" in project_dependencies +def test_wheel_contract_rejects_main_package_path_outside_venv(tmp_path: Path) -> None: + venv_dir = tmp_path / "venv" + installed_path = venv_dir / "Lib" / "site-packages" / "bt_api_py" / "__init__.py" + outside_path = tmp_path / "system" / "Lib" / "site-packages" / "bt_api_py" / "__init__.py" + nested_package_path = ( + venv_dir / "Lib" / "site-packages" / "unexpected" / "bt_api_py" / "__init__.py" + ) + + assert _require_venv_package_path(str(installed_path), venv_dir, "bt_api_py") == str( + installed_path.resolve() + ).replace("\\", "/") + with pytest.raises(WheelContractError, match="outside the virtualenv"): + _require_venv_package_path(str(outside_path), venv_dir, "bt_api_py") + with pytest.raises(WheelContractError, match="outside the virtualenv"): + _require_venv_package_path(str(nested_package_path), venv_dir, "bt_api_py") + + def test_wheel_contract_checker_runs_doctor_from_an_installed_wheel(tmp_path: Path) -> None: wheelhouse = build_project_wheelhouse(tmp_path / "wheelhouse") dist_dir = tmp_path / "dist" @@ -129,6 +151,30 @@ def test_wheel_contract_checker_runs_doctor_from_an_installed_wheel(tmp_path: Pa assert "site-packages/bt_api_py" in receipt["package_file"].replace("\\", "/") assert "site-packages/bt_api_base" in receipt["base_package_file"].replace("\\", "/") assert receipt["probe"]["base_package_file"] == receipt["base_package_file"] + base_source = receipt["base_source"] + source_gitlink = subprocess.check_output( + ["git", "ls-tree", "HEAD", "bt_api/bt_api_base"], # noqa: S607 + cwd=REPOSITORY_ROOT, + text=True, + ).split() + assert base_source["parent_commit"] == receipt["head_sha"] + assert base_source["source_commit"] == source_gitlink[2] + assert base_source["source_origin"] == BASE_ORIGIN + assert base_source["package_name"] == "bt_api_base" + assert base_source["package_version"] == "0.15.5" + assert base_source["wheel_sha256"] + base_wheel = dist_dir / base_source["wheel_path"] + assert base_source["wheel_path"] == (f"bt_api_base_source/{base_source['wheel_filename']}") + assert base_wheel.is_file() + assert base_source["wheel_path_url"] == base_wheel.resolve().as_uri() + assert receipt["probe"]["base_version"] == base_source["package_version"] + direct_url = receipt["probe"]["base_direct_url"] + assert direct_url["url"] == base_source["wheel_path_url"] + archive_info = direct_url["archive_info"] + recorded_hash = archive_info.get("hash") + if recorded_hash is None: + recorded_hash = f"sha256={archive_info['hashes']['sha256']}" + assert recorded_hash == f"sha256={base_source['wheel_sha256']}" def test_wheel_contract_rejects_invalid_wheelhouse_before_reading_artifacts( @@ -179,6 +225,34 @@ def test_ci_workflows_enforce_the_installed_wheel_contract() -> None: assert "scripts/ci/verify_wheel_contract.py" in tests_workflow tests_data = yaml.safe_load(tests_workflow) + wheel_contract_steps = tests_data["jobs"]["wheel-contract"]["steps"] + checkout_index = next( + index + for index, step in enumerate(wheel_contract_steps) + if step.get("uses", "").startswith("actions/checkout@") + ) + base_checkout_index = next( + index + for index, step in enumerate(wheel_contract_steps) + if step.get("name") == "Checkout parent-pinned base source" + ) + verifier_index = next( + index + for index, step in enumerate(wheel_contract_steps) + if step.get("name") == "Verify installed wheel resource contract" + ) + assert checkout_index < base_checkout_index < verifier_index + assert "submodules" not in wheel_contract_steps[checkout_index].get("with", {}) + assert ( + wheel_contract_steps[base_checkout_index]["run"] + == "git submodule update --init --depth 1 -- bt_api/bt_api_base" + ) + wheel_contract_artifact = next( + step + for step in wheel_contract_steps + if step.get("name") == "Archive wheel contract receipt" + ) + assert "dist/" in wheel_contract_artifact["with"]["path"].splitlines() full_suite_steps = tests_data["jobs"]["full-suite"]["steps"] full_suite_install = next( step for step in full_suite_steps if step.get("name") == "Install package + dev deps" diff --git a/tests/unit/workflows/test_base_source_pin_workflows.py b/tests/unit/workflows/test_base_source_pin_workflows.py new file mode 100644 index 00000000..a3d7c77c --- /dev/null +++ b/tests/unit/workflows/test_base_source_pin_workflows.py @@ -0,0 +1,105 @@ +"""Structural checks keep docs and test jobs on the exact parent source pin.""" + +from pathlib import Path + +import yaml + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +WORKFLOW_ROOT = REPOSITORY_ROOT / ".github" / "workflows" +BOOTSTRAP_COMMAND = "python scripts/ci/base_source_pin.py" + + +def _workflow_steps(filename: str, job: str) -> list[dict[str, object]]: + with (WORKFLOW_ROOT / filename).open(encoding="utf-8") as workflow_file: + workflow = yaml.load(workflow_file, Loader=yaml.BaseLoader) + return workflow["jobs"][job]["steps"] + + +def _step_index(steps: list[dict[str, object]], name: str) -> int: + return next(index for index, step in enumerate(steps) if step.get("name") == name) + + +def _assert_source_bootstrap_precedes_package_install( + steps: list[dict[str, object]], + install_step: str, + *, + expect_github_env: bool = False, +) -> None: + bootstrap_index = _step_index(steps, "Build and install parent-pinned base wheel") + install_index = _step_index(steps, install_step) + bootstrap = steps[bootstrap_index] + + assert bootstrap_index < install_index + assert BOOTSTRAP_COMMAND in bootstrap["run"] + assert '--wheel-dir "${{ runner.temp }}/bt_api_base_wheelhouse"' in bootstrap["run"] + assert "$RUNNER_TEMP" not in bootstrap["run"] + if expect_github_env: + assert '--github-env "$GITHUB_ENV"' in bootstrap["run"] + else: + assert "$GITHUB_ENV" not in bootstrap["run"] + assert ( + "continue-on-error" not in bootstrap + or str(bootstrap["continue-on-error"]).lower() != "true" + ) + assert "python -m pip check" in steps[_step_index(steps, "Check installed dependencies")]["run"] + + +def test_docs_job_checks_out_and_installs_parent_pinned_base_before_root_package() -> None: + steps = _workflow_steps("docs.yml", "build") + checkout_index = _step_index(steps, "Checkout pinned base source") + bootstrap_index = _step_index(steps, "Build and install parent-pinned base wheel") + install_index = _step_index(steps, "Install package (needed by mkdocstrings)") + + assert checkout_index < bootstrap_index < install_index + _assert_source_bootstrap_precedes_package_install( + steps, "Install package (needed by mkdocstrings)" + ) + + +def test_tests_workflow_bootstraps_every_root_dependency_install_job() -> None: + quality_steps = _workflow_steps("tests.yml", "quality") + full_suite_steps = _workflow_steps("tests.yml", "full-suite") + _assert_source_bootstrap_precedes_package_install( + quality_steps, "Install package + quality tools" + ) + _assert_source_bootstrap_precedes_package_install( + full_suite_steps, "Install package + dev deps", expect_github_env=True + ) + + +def test_windows_compatibility_job_uses_cross_platform_temp_and_only_checks_out_base() -> None: + steps = _workflow_steps("reusable-compat-matrix.yml", "matrix") + checkout_index = next( + index + for index, step in enumerate(steps) + if str(step.get("uses", "")).startswith("actions/checkout@") + ) + source_index = _step_index(steps, "Checkout pinned base source") + bootstrap_index = _step_index(steps, "Build and install parent-pinned base wheel") + checkout = steps[checkout_index] + source_checkout = steps[source_index] + + assert checkout_index < source_index < bootstrap_index + assert "submodules" not in checkout.get("with", {}) + assert source_checkout["run"] == "git submodule update --init --depth 1 -- bt_api/bt_api_base" + _assert_source_bootstrap_precedes_package_install(steps, "Install package + dev deps") + bootstrap_command = steps[bootstrap_index]["run"] + assert '"${{ runner.temp }}/bt_api_base_wheelhouse"' in bootstrap_command + assert "$RUNNER_TEMP" not in bootstrap_command + assert "$GITHUB_ENV" not in bootstrap_command + + +def test_codecov_secret_is_scoped_to_upload_and_fork_prs_skip_it() -> None: + with (WORKFLOW_ROOT / "tests.yml").open(encoding="utf-8") as workflow_file: + workflow = yaml.load(workflow_file, Loader=yaml.BaseLoader) + full_suite = workflow["jobs"]["full-suite"] + upload = next( + step for step in full_suite["steps"] if step.get("name") == "Upload coverage to Codecov" + ) + + assert "CODECOV_TOKEN" not in full_suite.get("env", {}) + assert upload["env"]["CODECOV_TOKEN"] == "${{ secrets.CODECOV_TOKEN }}" + assert upload["with"]["token"] == "${{ secrets.CODECOV_TOKEN }}" + assert "github.event_name != 'pull_request'" in upload["if"] + assert "github.event.pull_request.head.repo.full_name == github.repository" in upload["if"] + assert upload["with"]["fail_ci_if_error"] == "false" diff --git a/tests/unit/workflows/test_ctp_source_pin_workflows.py b/tests/unit/workflows/test_ctp_source_pin_workflows.py new file mode 100644 index 00000000..65b2436c --- /dev/null +++ b/tests/unit/workflows/test_ctp_source_pin_workflows.py @@ -0,0 +1,58 @@ +"""Structural CI checks for the parent-pinned CTP native source wheel.""" + +from pathlib import Path + +import yaml + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +WORKFLOW_PATH = REPOSITORY_ROOT / ".github" / "workflows" / "tests.yml" + + +def _steps() -> list[dict[str, object]]: + with WORKFLOW_PATH.open(encoding="utf-8") as workflow_file: + workflow = yaml.load(workflow_file, Loader=yaml.BaseLoader) + return workflow["jobs"]["full-suite"]["steps"] + + +def _index(steps: list[dict[str, object]], name: str) -> int: + return next(index for index, step in enumerate(steps) if step.get("name") == name) + + +def test_full_suite_installs_and_rechecks_exact_local_ctp_wheel() -> None: + steps = _steps() + base = _index(steps, "Build and install parent-pinned base wheel") + toolchain = _index(steps, "Install build tools") + ctp = _index(steps, "Build and install parent-pinned CTP source wheel") + package = _index(steps, "Install package + dev deps") + verify = _index(steps, "Verify parent-pinned CTP wheel receipt") + dependency_check = _index(steps, "Check installed dependencies") + native_check = _index(steps, "Check CTP native extension load") + + assert base < toolchain < ctp < package < verify < dependency_check < native_check + assert '--github-env "$GITHUB_ENV"' in steps[base]["run"] + assert "python scripts/ci/ctp_source_pin.py" in steps[ctp]["run"] + assert '--wheel-dir "${{ runner.temp }}/bt_api_ctp_wheelhouse"' in steps[ctp]["run"] + assert '--github-env "$GITHUB_ENV"' in steps[ctp]["run"] + assert steps[verify]["run"] == "python scripts/ci/ctp_source_pin.py --verify-installed" + assert "python -m pip check" in steps[dependency_check]["run"] + assert "is_ctp_native_loaded()" in steps[native_check]["run"] + assert "CODECOV_TOKEN" not in steps[ctp].get("env", {}) + assert "GITHUB_TOKEN" not in steps[ctp].get("env", {}) + assert "env" not in steps[ctp] + with WORKFLOW_PATH.open(encoding="utf-8") as workflow_file: + workflow = yaml.load(workflow_file, Loader=yaml.BaseLoader) + full_suite_job = workflow["jobs"]["full-suite"] + assert "env" not in full_suite_job + assert "continue-on-error" not in steps[ctp] + + +def test_ctp_bootstrap_uses_only_parent_pin_and_shared_strict_environment() -> None: + script = (REPOSITORY_ROOT / "scripts" / "ci" / "ctp_source_pin.py").read_text(encoding="utf-8") + assert '"archive"' in script + assert '"--format=tar"' in script + assert "pin.source_commit" in script + assert '"--no-deps"' in script + assert "_subprocess_environment(public_pip_index=True)" in script + assert "_subprocess_environment()" in script + assert "verify_ctp_source_pin(repository_root)" in script + assert "source_commit" in script diff --git a/tests/unit/workflows/test_tests_workflow.py b/tests/unit/workflows/test_tests_workflow.py index 9b8463dc..b15d795e 100644 --- a/tests/unit/workflows/test_tests_workflow.py +++ b/tests/unit/workflows/test_tests_workflow.py @@ -256,7 +256,10 @@ def test_coverage_reports_are_validated_before_optional_uploads() -> None: codecov_options = _mapping( _required_field(codecov_step, "with", codecov_path), f"{codecov_path}.with" ) - assert _string_field(codecov_step, "if", codecov_path) == "always() && env.CODECOV_TOKEN != ''" + assert _string_field(codecov_step, "if", codecov_path) == ( + "always() && (github.event_name != 'pull_request' || " + "github.event.pull_request.head.repo.full_name == github.repository)" + ) assert _string_field(codecov_options, "fail_ci_if_error", f"{codecov_path}.with") == "false"