From 37d9ab7c193396b6b31dafee28635e3e8e6c3989 Mon Sep 17 00:00:00 2001 From: Jake Fineman Date: Sat, 5 Sep 2026 14:46:16 -0400 Subject: [PATCH 1/2] ci(ga): GA evidence producer for VER-001 + SUPPLY-001 Adds the E1 GA-readiness-gate evidence producer for this repo, on the wave-av/sdks registry-clean-room pattern. Verifies what PyPI and GitHub actually serve (never the checkout) and writes ga-out/wave-av__sdk-python.ga-evidence.json. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ga-evidence.yml | 99 +++++++++++++++++ .gitignore | 10 ++ scripts/ga/check-SUPPLY-001.sh | 29 +++++ scripts/ga/check-VER-001.sh | 34 ++++++ scripts/ga/check_supply_001.py | 69 ++++++++++++ scripts/ga/check_ver_001.py | 171 ++++++++++++++++++++++++++++++ scripts/ga/ga_common.py | 166 +++++++++++++++++++++++++++++ scripts/ga/ga_evidence.py | 115 ++++++++++++++++++++ 8 files changed, 693 insertions(+) create mode 100644 .github/workflows/ga-evidence.yml create mode 100644 .gitignore create mode 100755 scripts/ga/check-SUPPLY-001.sh create mode 100755 scripts/ga/check-VER-001.sh create mode 100644 scripts/ga/check_supply_001.py create mode 100644 scripts/ga/check_ver_001.py create mode 100644 scripts/ga/ga_common.py create mode 100755 scripts/ga/ga_evidence.py diff --git a/.github/workflows/ga-evidence.yml b/.github/workflows/ga-evidence.yml new file mode 100644 index 0000000..ef09361 --- /dev/null +++ b/.github/workflows/ga-evidence.yml @@ -0,0 +1,99 @@ +name: ga evidence + +# Producer for the WAVE GA readiness gate — VER-001 and SUPPLY-001 — computed against what the +# PUBLIC PyPI registry and GitHub actually serve, never the checkout under test. See +# scripts/ga/ga_evidence.py for what each criterion verifies and what it leaves `unknown`. +# +# wave-av/sdks is the only other repo in the WAVE org that ships a GA-evidence producer today +# (its `registry clean-room acceptance` workflow). This mirrors that repo's fail-loud posture: +# every trigger reports its true state, no `|| true`, no continue-on-error, and the final +# Enforce step turns a non-zero producer exit into a red job. +# +# `pull_request` legitimately sees VER-001 as `unknown` on a release PR whose tag/version is +# ahead of what PyPI has published — the producer reports that as `unknown`, not `fail`; see the +# HEAD-ahead-of-published branch in scripts/ga/check_ver_001.py. + +on: + pull_request: + workflow_dispatch: + inputs: + expect_version: + description: 'Assert PyPI now serves exactly this version (e.g. a release job verifying its own publish)' + type: string + required: false + schedule: + # 09:43 UTC — offset from a round hour so a registry rate-limit window shared across the org's + # scheduled jobs does not land on this one every day. + - cron: "43 9 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + ga-evidence: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Run GA evidence producer against the public registries + id: evidence + env: + GA_EXPECT_VERSION: ${{ inputs.expect_version }} + run: | + set -uo pipefail + args=(--out-dir "$GITHUB_WORKSPACE/ga-out") + [ -n "${GA_EXPECT_VERSION:-}" ] && args+=(--expect-version "$GA_EXPECT_VERSION") + set +e + python3 scripts/ga/ga_evidence.py "${args[@]}" 2>&1 | tee "$RUNNER_TEMP/ga-evidence.log" + code=${PIPESTATUS[0]} + set -e + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + { + echo "## GA evidence — VER-001 / SUPPLY-001" + echo + echo "Exit code \`$code\` (0 = pass/unknown, 1 = a criterion failed, 2 = the producer could not run)." + echo + echo '```' + cat "$RUNNER_TEMP/ga-evidence.log" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Upload GA evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ga-evidence-sdk-python + path: ga-out/ + if-no-files-found: warn + retention-days: 90 + + - name: Enforce + # A gate that cannot fail is not a gate (wave-av/sdks#79 is the org's own cautionary + # tale — see registry-cleanroom.yml). This step fails loud on every trigger, including + # pull_request; whether that failure is a *required* branch-protection check is a + # separate branch-ruleset decision, not something this workflow should paper over by + # exiting 0 on a red producer run. + env: + CODE: ${{ steps.evidence.outputs.exit_code }} + run: | + if [ "$CODE" = "1" ]; then + echo "::error title=ga-evidence::a GA criterion failed verification (exit 1) — see the job summary" + exit 1 + fi + if [ "$CODE" != "0" ]; then + echo "::error title=ga-evidence::the producer could not run (exit $CODE) — never read as a pass" + exit 1 + fi + echo "ga-evidence: no criterion failed (pass or unknown only) — see the job summary for detail" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..05b30b1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +# GA evidence producer output — regenerated by scripts/ga/ga_evidence.py, never committed. +ga-out/ + +# Python +__pycache__/ +*.pyc +.venv/ +dist/ +build/ +*.egg-info/ diff --git a/scripts/ga/check-SUPPLY-001.sh b/scripts/ga/check-SUPPLY-001.sh new file mode 100755 index 0000000..ee1ee12 --- /dev/null +++ b/scripts/ga/check-SUPPLY-001.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# SUPPLY-001 — release artifacts built by approved CI from an immutable source revision, +# provenance verifiable, SBOM attached. +# +# Thin wrapper around ga_evidence.py (see there for what is and is not machine-verified: the +# provenance clause only — SBOM attachment and known-vuln resolution are named as unverified, +# never assumed). This script filters the shared run down to the SUPPLY-001 line so it can also +# be invoked standalone. +# +# Prints one `PASS|FAIL|UNKNOWN SUPPLY-001: ` line. +# Exit 0 = pass, 1 = fail, 2 = could not run (never read as a pass). +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="${GA_OUT_DIR:-$HERE/../../ga-out}" +REPO="${GA_REPO:-wave-av/sdk-python}" +PACKAGE="${GA_PACKAGE:-wave-sdk}" + +OUTPUT="$(python3 "$HERE/ga_evidence.py" --out-dir "$OUT_DIR" --repo "$REPO" --package "$PACKAGE" 2>&1)" +CODE=$? + +echo "$OUTPUT" | grep -E '^(PASS|FAIL|UNKNOWN) SUPPLY-001:' +if [ "$CODE" -eq 2 ]; then + echo "$OUTPUT" 1>&2 + exit 2 +fi + +echo "$OUTPUT" | grep -q '^FAIL SUPPLY-001:' && exit 1 +exit 0 diff --git a/scripts/ga/check-VER-001.sh b/scripts/ga/check-VER-001.sh new file mode 100755 index 0000000..e3235cc --- /dev/null +++ b/scripts/ga/check-VER-001.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# VER-001 — every shipped component resolves to one source revision and version; no newer +# source is represented as deployed. +# +# Thin wrapper around ga_evidence.py, which computes both criteria in one registry-fetch pass +# (VER-001 and SUPPLY-001 share the same PyPI `info.version` lookup). This script filters the +# shared run down to the VER-001 line so it can also be invoked standalone. +# +# Prints one `PASS|FAIL|UNKNOWN VER-001: ` line. +# Exit 0 = pass, 1 = fail, 2 = could not run (never read as a pass). +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="${GA_OUT_DIR:-$HERE/../../ga-out}" +REPO="${GA_REPO:-wave-av/sdk-python}" +PACKAGE="${GA_PACKAGE:-wave-sdk}" + +ARGS=(--out-dir "$OUT_DIR" --repo "$REPO" --package "$PACKAGE") +# GA_EXPECT_VERSION: optional pin asserting PyPI now serves exactly this version (e.g. a release +# job verifying its own publish). Also the deliberate-break lever for the drill this producer's +# PR must prove: pin a wrong version and this check flips PASS/UNKNOWN -> FAIL, exit 1. +[ -n "${GA_EXPECT_VERSION:-}" ] && ARGS+=(--expect-version "$GA_EXPECT_VERSION") + +OUTPUT="$(python3 "$HERE/ga_evidence.py" "${ARGS[@]}" 2>&1)" +CODE=$? + +echo "$OUTPUT" | grep -E '^(PASS|FAIL|UNKNOWN) VER-001:' +if [ "$CODE" -eq 2 ]; then + echo "$OUTPUT" 1>&2 + exit 2 +fi + +echo "$OUTPUT" | grep -q '^FAIL VER-001:' && exit 1 +exit 0 diff --git a/scripts/ga/check_supply_001.py b/scripts/ga/check_supply_001.py new file mode 100644 index 0000000..cbc5906 --- /dev/null +++ b/scripts/ga/check_supply_001.py @@ -0,0 +1,69 @@ +"""SUPPLY-001 — release artifacts built by approved CI from an immutable source revision, +provenance verifiable, SBOM attached. + +This producer machine-verifies the provenance clause ONLY: the PyPI Integrity API is queried for +every published artifact (wheel + sdist), and the attestation's claimed source repository must be +`github.com/`. SBOM attachment and critical-vulnerability resolution are NOT machine-verified +here, so a fully-verified provenance still yields `unknown` (never `pass`) with those two gaps +named explicitly in `failing_checks`. Absent or mismatched provenance is `fail`. +""" +from __future__ import annotations + +import json + +from ga_common import CheckResult, CriterionResult, fetch_json, fetch_json_allow_404, pypi_url + + +def run(repo: str, package: str) -> CriterionResult: + command = f"python3 scripts/ga/ga_evidence.py --repo {repo} --package {package}" + checks: list[CheckResult] = [] + + meta = fetch_json(pypi_url(package)) + version = meta["info"]["version"] + urls = meta.get("urls", []) + targets = [f"{package}@{version}"] + + if not urls: + checks.append(CheckResult("pypi-artifacts-present", False, f"PyPI serves no files for {package}=={version}")) + return CriterionResult("SUPPLY-001", "fail", command, checks, targets) + + all_have_provenance = True + wrong_repo_claims: list[str] = [] + for u in urls: + filename = u["filename"] + prov_url = ( + f"https://pypi.org/integrity/{package}/{version}/{filename}/provenance" + ) + status, body = fetch_json_allow_404(prov_url) + if status == 404 or body is None or "attestation_bundles" not in body: + all_have_provenance = False + checks.append(CheckResult( + f"provenance-present:{filename}", False, + "no provenance available from PyPI Integrity API", + )) + continue + bundles = body.get("attestation_bundles", []) + raw = json.dumps(body) + repo_claim_ok = f"github.com/{repo}" in raw or repo in raw + if repo_claim_ok: + checks.append(CheckResult( + f"provenance-present:{filename}", True, + f"PyPI Integrity API returned {len(bundles)} attestation bundle(s) referencing {repo}", + )) + else: + wrong_repo_claims.append(filename) + checks.append(CheckResult( + f"provenance-present:{filename}", False, + f"attestation present but does not reference {repo}", + )) + + if wrong_repo_claims or not all_have_provenance: + status = "fail" + else: + # Provenance verifies for every artifact, but SBOM attachment and known-vuln resolution + # stay out of scope for this producer — the criterion cannot be a full pass. + status = "unknown" + checks.append(CheckResult("sbom-attached", None, "SBOM attachment not verified by this producer")) + checks.append(CheckResult("known-vuln-resolution", None, "critical-vuln resolution not verified by this producer")) + + return CriterionResult("SUPPLY-001", status, command, checks, targets) diff --git a/scripts/ga/check_ver_001.py b/scripts/ga/check_ver_001.py new file mode 100644 index 0000000..8c46925 --- /dev/null +++ b/scripts/ga/check_ver_001.py @@ -0,0 +1,171 @@ +"""VER-001 — every shipped component resolves to one source revision and version; no newer +source is represented as deployed. + +Compares: HEAD's pyproject.toml version, PyPI's published `info.version`, the newest `v*` tag on +GitHub, that tag's GitHub Release (if any), and the published wheel's METADATA `Version`. `pass` +only when all agree. HEAD legitimately running ahead of PyPI (an open release PR) is `unknown` +("unreleased source"), never `fail`. +""" +from __future__ import annotations + +import hashlib + +from ga_common import ( + CheckResult, + CriterionResult, + fetch_bytes, + fetch_json, + fetch_json_allow_404, + pypi_url, + read_head_version, + semver_tuple, + status_from_checks, + wheel_metadata_version, +) + + +def run(repo: str, package: str, expect_version: str | None = None) -> CriterionResult: + command = f"python3 scripts/ga/ga_evidence.py --repo {repo} --package {package}" + if expect_version: + command += f" --expect-version {expect_version}" + checks: list[CheckResult] = [] + targets: list[str] = [] + + head_version = read_head_version() + meta = fetch_json(pypi_url(package)) + pypi_version = meta["info"]["version"] + targets.append(f"{package}@{pypi_version}") + + # Optional pin, analogous to the sdks reference producer's `--versions` pin: a release job + # can assert the exact version it just published is what PyPI now serves. Never required — + # only present when the caller (or GA_EXPECT_VERSION) supplies one. + if expect_version is not None: + if expect_version == pypi_version: + checks.append(CheckResult( + "expected-version-matches-published", True, + f"expected version {expect_version} == published PyPI version {pypi_version}", + )) + else: + checks.append(CheckResult( + "expected-version-matches-published", False, + f"expected version {expect_version} != published PyPI version {pypi_version}", + )) + + urls = meta.get("urls", []) + wheel_url = next((u for u in urls if u.get("packagetype") == "bdist_wheel"), None) + if wheel_url is None: + checks.append(CheckResult("pypi-wheel-present", False, f"PyPI serves no wheel for {package}=={pypi_version}")) + else: + wheel_bytes = fetch_bytes(wheel_url["url"]) + declared_sha = wheel_url.get("digests", {}).get("sha256") + actual_sha = hashlib.sha256(wheel_bytes).hexdigest() + if declared_sha and declared_sha != actual_sha: + checks.append(CheckResult( + "wheel-digest-matches-index", False, + f"downloaded {wheel_url['filename']} sha256 {actual_sha} != PyPI-declared {declared_sha}", + )) + else: + checks.append(CheckResult( + "wheel-digest-matches-index", True, + f"downloaded {wheel_url['filename']} sha256 matches the PyPI-declared digest", + )) + wheel_metadata = wheel_metadata_version(wheel_bytes, wheel_url["filename"]) + if wheel_metadata == pypi_version: + checks.append(CheckResult( + "wheel-metadata-matches-pypi-version", True, + f"wheel METADATA Version {wheel_metadata} == PyPI info.version {pypi_version}", + )) + else: + checks.append(CheckResult( + "wheel-metadata-matches-pypi-version", False, + f"wheel METADATA Version {wheel_metadata} != PyPI info.version {pypi_version}", + )) + + # Newest v* tag and its GitHub release, via the public GitHub API (unauthenticated is fine — + # this reads public tag/release metadata, never the checkout). + _, tags = fetch_json_allow_404(f"https://api.github.com/repos/{repo}/tags?per_page=100") + tag_versions: list[tuple[tuple[int, int, int], str]] = [] + for t in (tags or []): + name = t.get("name", "") + if name.startswith("v"): + sv = semver_tuple(name[1:]) + if sv: + tag_versions.append((sv, name)) + + newest_tag_version = None + if not tag_versions: + checks.append(CheckResult("newest-tag-exists", None, "no v* semver tags found on origin")) + else: + tag_versions.sort() + newest_tag = tag_versions[-1][1] + newest_tag_version = newest_tag[1:] + checks.append(CheckResult("newest-tag-exists", True, f"newest v* tag is {newest_tag}")) + + rel_status, release = fetch_json_allow_404(f"https://api.github.com/repos/{repo}/releases/tags/{newest_tag}") + if rel_status == 404: + checks.append(CheckResult( + "newest-tag-has-github-release", None, + f"no GitHub Release object exists for tag {newest_tag}", + )) + elif release and release.get("tag_name") == newest_tag: + checks.append(CheckResult( + "newest-tag-has-github-release", True, + f"GitHub Release for {newest_tag} exists and its tag_name matches", + )) + else: + checks.append(CheckResult( + "newest-tag-has-github-release", False, + f"GitHub Release for {newest_tag} has tag_name {release.get('tag_name') if release else None!r}", + )) + + # HEAD vs published: HEAD may legitimately be ahead of PyPI on a pull_request (a release PR + # that has not published yet) — that is `unknown`, not `fail`. + head_sv = semver_tuple(head_version) + pypi_sv = semver_tuple(pypi_version) + if head_sv is not None and pypi_sv is not None: + if head_sv == pypi_sv: + checks.append(CheckResult( + "head-version-matches-published", True, + f"HEAD pyproject.toml version {head_version} == published PyPI version {pypi_version}", + )) + elif head_sv > pypi_sv: + checks.append(CheckResult( + "head-version-matches-published", None, + f"HEAD pyproject.toml version {head_version} is ahead of published PyPI version " + f"{pypi_version} — unreleased source, not yet represented as deployed", + )) + else: + checks.append(CheckResult( + "head-version-matches-published", False, + f"HEAD pyproject.toml version {head_version} is BEHIND published PyPI version " + f"{pypi_version} — a newer source is represented as deployed than is checked out", + )) + else: + checks.append(CheckResult( + "head-version-matches-published", False, + f"could not parse semver from HEAD version {head_version!r} or PyPI version {pypi_version!r}", + )) + + # Newest tag vs published version. + if newest_tag_version is not None: + newest_tag_sv = semver_tuple(newest_tag_version) + if newest_tag_sv is not None and pypi_sv is not None: + if newest_tag_sv == pypi_sv: + checks.append(CheckResult( + "newest-tag-matches-published", True, + f"newest tag version {newest_tag_version} == published PyPI version {pypi_version}", + )) + elif newest_tag_sv > pypi_sv: + checks.append(CheckResult( + "newest-tag-matches-published", None, + f"tag v{newest_tag_version} exists but PyPI still serves {pypi_version} — release " + f"pending or the publish step has not completed for this tag", + )) + else: + checks.append(CheckResult( + "newest-tag-matches-published", False, + f"PyPI serves {pypi_version} but the newest recorded tag is only " + f"{newest_tag_version} — the published version has no corresponding source tag", + )) + + return CriterionResult("VER-001", status_from_checks(checks), command, checks, sorted(set(targets))) diff --git a/scripts/ga/ga_common.py b/scripts/ga/ga_common.py new file mode 100644 index 0000000..27f5124 --- /dev/null +++ b/scripts/ga/ga_common.py @@ -0,0 +1,166 @@ +"""Shared primitives for the GA evidence producer: registry fetch, semver, and the evidence +document shape. Stdlib only. Nothing here reads the checkout for anything registry-shaped — +only `read_head_version()` and `git_head_sha()` do, and both are read-only. +""" +from __future__ import annotations + +import hashlib +import io +import json +import re +import subprocess +import urllib.error +import urllib.request +import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from urllib.parse import quote + +import tomllib + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +USER_AGENT = "wave-ga-evidence-sdk-python/1.0" +SEMVER_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)") + + +class RegistryError(RuntimeError): + """Raised when a public registry cannot be reached — always exit 2, never a pass.""" + + +def semver_tuple(v: str) -> tuple[int, int, int] | None: + m = SEMVER_RE.match(v.strip()) + if not m: + return None + return (int(m.group(1)), int(m.group(2)), int(m.group(3))) + + +def fetch_json(url: str, timeout: int = 30) -> dict: + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, json.JSONDecodeError) as e: + raise RegistryError(f"GET {url} failed: {type(e).__name__}: {e}") from e + + +def fetch_json_allow_404(url: str, timeout: int = 30) -> tuple[int, dict | None]: + """Like fetch_json but a 404 is a normal, expected outcome — not a registry failure.""" + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + if e.code == 404: + return 404, None + raise RegistryError(f"GET {url} failed: HTTP {e.code}") from e + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as e: + raise RegistryError(f"GET {url} failed: {type(e).__name__}: {e}") from e + + +def fetch_bytes(url: str, timeout: int = 60) -> bytes: + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.read() + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as e: + raise RegistryError(f"GET {url} failed: {type(e).__name__}: {e}") from e + + +def pypi_url(package: str, suffix: str = "") -> str: + base = f"https://pypi.org/pypi/{quote(package)}/json" + return base if not suffix else f"{base}/{suffix}" + + +def wheel_metadata_version(wheel_bytes: bytes, filename: str) -> str: + with zipfile.ZipFile(io.BytesIO(wheel_bytes)) as zf: + metadata_names = [n for n in zf.namelist() if n.endswith(".dist-info/METADATA")] + if not metadata_names: + raise RegistryError(f"{filename}: no *.dist-info/METADATA member in wheel") + text = zf.read(metadata_names[0]).decode("utf-8", errors="replace") + for line in text.splitlines(): + if line.startswith("Version:"): + return line.split(":", 1)[1].strip() + raise RegistryError(f"{filename}: METADATA has no Version: field") + + +def read_head_version() -> str: + data = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text("utf-8")) + return data["project"]["version"] + + +def git_head_sha() -> str: + r = subprocess.run(["git", "rev-parse", "HEAD"], cwd=REPO_ROOT, capture_output=True, text=True, timeout=30) + if r.returncode != 0: + raise RegistryError(f"git rev-parse HEAD failed: {r.stderr.strip()}") + return r.stdout.strip() + + +@dataclass +class CheckResult: + name: str + ok: bool | None # True=pass-contributing, False=fail-contributing, None=unknown-contributing + detail: str + + +@dataclass +class CriterionResult: + criterion_id: str + status: str # pass | fail | unknown + command: str + checks: list[CheckResult] = field(default_factory=list) + targets_observed: list[str] = field(default_factory=list) + + +def status_from_checks(checks: list[CheckResult]) -> str: + ok_vals = [c.ok for c in checks] + if any(v is False for v in ok_vals): + return "fail" + if any(v is None for v in ok_vals): + return "unknown" + return "pass" + + +def canonical_fingerprint_input(results: list[CriterionResult]) -> dict: + """Fingerprint input: criterion ids, check names, ok flags, observed versions/digests. + Deliberately excludes timestamps, temp paths and durations so two runs against the same + published artifacts produce the same digest.""" + rows = [] + for r in sorted(results, key=lambda r: r.criterion_id): + rows.append({ + "criterion_id": r.criterion_id, + "status": r.status, + "targets_observed": sorted(r.targets_observed), + "checks": sorted([[c.name, c.ok] for c in r.checks]), + }) + return {"rows": rows} + + +def sha256_canonical(obj: dict) -> str: + blob = json.dumps(obj, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +def build_document(repo: str, revision: str, results: list[CriterionResult], verified_at: str, fingerprint: str) -> dict: + out_results = [] + for r in sorted(results, key=lambda r: r.criterion_id): + entry = { + "criterion_id": r.criterion_id, + "status": r.status, + "command": r.command, + "evidence_sha256": fingerprint, + "evidence_uri": "ci://wave-av/sdk-python/.github/workflows/ga-evidence.yml#ga-report.json", + "verified_at": verified_at, + } + if r.targets_observed: + entry["targets_observed"] = r.targets_observed + failing = [f"{c.name}: {c.detail}" for c in r.checks if c.ok is False] + \ + [f"{c.name}: {c.detail}" for c in r.checks if c.ok is None] + if failing: + entry["failing_checks"] = failing + out_results.append(entry) + return { + "spec_version": "1.0.0", + "repository": repo, + "revision": revision, + "results": out_results, + } diff --git a/scripts/ga/ga_evidence.py b/scripts/ga/ga_evidence.py new file mode 100755 index 0000000..6cdab1e --- /dev/null +++ b/scripts/ga/ga_evidence.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""GA evidence producer for wave-av/sdk-python — VER-001 and SUPPLY-001. + +Verifies what the PUBLIC PyPI registry and GitHub actually serve, never the checkout under test +for anything registry-shaped (the checkout only supplies HEAD's own pyproject.toml version and +the git revision, both read-only). See check_ver_001.py and check_supply_001.py for the criteria +themselves; this module is the thin orchestrator: run both, write the two output files, print one +PASS|FAIL|UNKNOWN line per criterion, and set the process exit code. + +OUTPUT + /ga-report.json full detail: every observed version/digest/check + /wave-av__sdk-python.ga-evidence.json WAVE-GA-gate-spec-v1.0.0 evidence document + +EXIT CODES + 0 every criterion is pass or unknown (a criterion that legitimately cannot fully pass yet + still lets the job succeed; only a real defect or a broken run should redden CI) + 1 at least one criterion is fail + 2 the gate could not run (a registry fetch failed, or similar) — never read as a pass + +USAGE + python3 scripts/ga/ga_evidence.py [--out-dir DIR] [--repo OWNER/NAME] [--package NAME] +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import check_supply_001 +import check_ver_001 +from ga_common import ( + RegistryError, + build_document, + canonical_fingerprint_input, + git_head_sha, + sha256_canonical, +) + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--out-dir", default=str(REPO_ROOT / "ga-out")) + ap.add_argument("--repo", default="wave-av/sdk-python") + ap.add_argument("--package", default="wave-sdk") + ap.add_argument( + "--expect-version", + default=None, + help="assert this exact version is what PyPI now serves (e.g. a release job verifying " + "its own publish); defaults to $GA_EXPECT_VERSION, unset means no assertion", + ) + args = ap.parse_args() + expect_version = args.expect_version or os.environ.get("GA_EXPECT_VERSION") or None + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + try: + revision = git_head_sha() + ver = check_ver_001.run(args.repo, args.package, expect_version=expect_version) + supply = check_supply_001.run(args.repo, args.package) + except RegistryError as e: + sys.stderr.write(f"ga-evidence could not run: {e}\n") + return 2 + + results = [ver, supply] + fingerprint = sha256_canonical(canonical_fingerprint_input(results)) + verified_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + document = build_document(args.repo, revision, results, verified_at, fingerprint) + + report = { + "schema": "wave-ga-evidence-sdk-python/1", + "spec_version": "1.0.0", + "repository": args.repo, + "revision": revision, + "generated_at": verified_at, + "evidence_sha256": fingerprint, + "criteria": [ + { + "criterion_id": r.criterion_id, + "status": r.status, + "command": r.command, + "targets_observed": r.targets_observed, + "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail} for c in r.checks], + } + for r in results + ], + } + + (out_dir / "ga-report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + (out_dir / "wave-av__sdk-python.ga-evidence.json").write_text( + json.dumps(document, indent=2) + "\n", encoding="utf-8" + ) + + exit_code = 0 + for r in results: + line_status = {"pass": "PASS", "fail": "FAIL", "unknown": "UNKNOWN"}[r.status] + detail = "; ".join(f"{c.name}={c.ok}" for c in r.checks) + print(f"{line_status} {r.criterion_id}: {detail}") + if r.status == "fail": + exit_code = 1 + + print(f"\nevidence fingerprint: {fingerprint}") + print(f"wrote {out_dir / 'ga-report.json'} and {out_dir / 'wave-av__sdk-python.ga-evidence.json'}") + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) From 2431f38e85420798557bf9d03e45db442c66eae6 Mon Sep 17 00:00:00 2001 From: Jake Fineman Date: Sat, 5 Sep 2026 17:56:02 -0400 Subject: [PATCH 2/2] ci(ga-evidence): PR job warns on live criterion failure, fails only when the producer cannot run Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ga-evidence.yml | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ga-evidence.yml b/.github/workflows/ga-evidence.yml index ef09361..263bb39 100644 --- a/.github/workflows/ga-evidence.yml +++ b/.github/workflows/ga-evidence.yml @@ -12,6 +12,10 @@ name: ga evidence # `pull_request` legitimately sees VER-001 as `unknown` on a release PR whose tag/version is # ahead of what PyPI has published — the producer reports that as `unknown`, not `fail`; see the # HEAD-ahead-of-published branch in scripts/ga/check_ver_001.py. +# +# PR CONTRACT: on `pull_request`, exit 1 (a live criterion failed) is a `::warning`, not a job +# failure — that is a property of the live registry, not of the PR's diff. Exit 2 (the producer +# could not run) always fails the job, and on schedule/workflow_dispatch/push exit 1 fails it too. on: pull_request: @@ -81,19 +85,26 @@ jobs: - name: Enforce # A gate that cannot fail is not a gate (wave-av/sdks#79 is the org's own cautionary - # tale — see registry-cleanroom.yml). This step fails loud on every trigger, including - # pull_request; whether that failure is a *required* branch-protection check is a - # separate branch-ruleset decision, not something this workflow should paper over by - # exiting 0 on a red producer run. + # tale — see registry-cleanroom.yml). This step fails loud on every trigger except one: + # see the PR CONTRACT note in the header — a live-criterion failure (exit 1) on + # `pull_request` logs a `::warning` and exits 0 instead of failing the job, because that + # failure is a property of the live registry, not of this PR's diff. Whether the job is a + # *required* branch-protection check is a separate branch-ruleset decision. env: CODE: ${{ steps.evidence.outputs.exit_code }} + EVENT: ${{ github.event_name }} run: | + if [ "$CODE" = "0" ]; then + echo "ga-evidence: no criterion failed (pass or unknown only) — see the job summary for detail" + exit 0 + fi + if [ "$CODE" = "1" ] && [ "$EVENT" = "pull_request" ]; then + echo "::warning title=ga-evidence::sdk-python GA evidence producer reports a failing live criterion (exit 1); evidence is in the job summary and artifact; this does not fail the PR because the criterion is a property of the live surface, not of this change" + exit 0 + fi if [ "$CODE" = "1" ]; then echo "::error title=ga-evidence::a GA criterion failed verification (exit 1) — see the job summary" exit 1 fi - if [ "$CODE" != "0" ]; then - echo "::error title=ga-evidence::the producer could not run (exit $CODE) — never read as a pass" - exit 1 - fi - echo "ga-evidence: no criterion failed (pass or unknown only) — see the job summary for detail" + echo "::error title=ga-evidence::the producer could not run (exit $CODE) — never read as a pass" + exit 1