R2 engineering program: trust-chain PRs, profiles, holdout, and rc.1 docs

.github/workflows/holdout-eval.yml

@@ -4,6 +4,7 @@ name: FormalPR-Holdout eval
 # Requires repository secret HOLDOUT_DOWNLOAD_TOKEN with read access to
 # fraware/FormalPR-Holdout private release assets. The independently recorded
 # asset SHA-256 is required so a mutable/replaced release asset cannot execute.
+# Prefer workflow input asset_sha256; otherwise repository variable HOLDOUT_ASSET_SHA256.
 on:
   workflow_dispatch:
     inputs:
@@ -12,9 +13,10 @@ on:
         required: true
         default: v0.1.0-synthetic
         type: string
-      holdout_asset_sha256:
-        description: Independently recorded SHA-256 of the frozen release asset
-        required: true
+      asset_sha256:
+        description: Immutable SHA-256 of the release asset (overrides repo variable)
+        required: false
+        default: ""
         type: string
       predictions_artifact:
         description: Optional name of a workflow artifact containing predictions.json
@@ -33,13 +35,19 @@ jobs:
         id: gate
         env:
           HOLDOUT_DOWNLOAD_TOKEN: ${{ secrets.HOLDOUT_DOWNLOAD_TOKEN }}
+          HOLDOUT_ASSET_SHA256: ${{ inputs.asset_sha256 || vars.HOLDOUT_ASSET_SHA256 }}
         run: |
           if [ -z "$HOLDOUT_DOWNLOAD_TOKEN" ]; then
             echo "configured=false" >> "$GITHUB_OUTPUT"
             echo "FormalPR-Holdout eval skipped: HOLDOUT_DOWNLOAD_TOKEN not configured."
             echo "See docs/FORMALPR_HOLDOUT_GOVERNANCE.md"
+          elif [ -z "$HOLDOUT_ASSET_SHA256" ]; then
+            echo "configured=false" >> "$GITHUB_OUTPUT"
+            echo "FormalPR-Holdout eval skipped: HOLDOUT_ASSET_SHA256 not configured."
+            echo "Provide workflow input asset_sha256 or repository variable HOLDOUT_ASSET_SHA256."
           else
             echo "configured=true" >> "$GITHUB_OUTPUT"
+            echo "asset_sha256=$HOLDOUT_ASSET_SHA256" >> "$GITHUB_OUTPUT"
           fi
 
       - uses: actions/checkout@v4
@@ -50,12 +58,6 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install OVK runner dependencies
-        if: steps.gate.outputs.configured == 'true'
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e .
-
       - name: Require predictions file
         if: steps.gate.outputs.configured == 'true'
         id: preds
@@ -87,35 +89,61 @@ jobs:
           fi
           test -f .verification/holdout-predictions.json
 
-      - name: Run holdout aggregate eval (fail closed)
+      - name: Download holdout asset (token only here)
         if: steps.gate.outputs.configured == 'true'
+        id: download
         env:
           HOLDOUT_DOWNLOAD_TOKEN: ${{ secrets.HOLDOUT_DOWNLOAD_TOKEN }}
-          HOLDOUT_ASSET_SHA256: ${{ inputs.holdout_asset_sha256 }}
         run: |
+          python - <<'PY'
+          import os
+          from pathlib import Path
+          from scripts.run_formalpr_holdout import download_release_asset, verify_asset_sha256
+
+          tag = "${{ inputs.holdout_tag }}"
+          asset_name = f"FormalPR-Holdout-{tag}.tar.gz"
+          dest = Path(".verification") / asset_name
+          dest.parent.mkdir(parents=True, exist_ok=True)
+          download_release_asset(
+              repo="fraware/FormalPR-Holdout",
+              tag=tag,
+              asset_name=asset_name,
+              dest=dest,
+              token=os.environ["HOLDOUT_DOWNLOAD_TOKEN"],
+          )
+          verify_asset_sha256(dest, "${{ steps.gate.outputs.asset_sha256 }}")
+          print(f"downloaded and verified {dest}")
+          PY
+          echo "artifact=.verification/FormalPR-Holdout-${{ inputs.holdout_tag }}.tar.gz" >> "$GITHUB_OUTPUT"
+
+      - name: Run holdout aggregate eval (fail closed, no tokens)
+        if: steps.gate.outputs.configured == 'true'
+        env:
+          # Explicitly unset download tokens for the evaluator process.
+          HOLDOUT_DOWNLOAD_TOKEN: ""
+          GITHUB_TOKEN: ""
+          GH_TOKEN: ""
+        run: |
+          unset HOLDOUT_DOWNLOAD_TOKEN GITHUB_TOKEN GH_TOKEN || true
           python scripts/run_formalpr_holdout.py \
+            --artifact "${{ steps.download.outputs.artifact }}" \
+            --asset-sha256 "${{ steps.gate.outputs.asset_sha256 }}" \
             --tag "${{ inputs.holdout_tag }}" \
-            --asset-sha256 "${HOLDOUT_ASSET_SHA256}" \
             --predictions .verification/holdout-predictions.json \
             --ovk-commit-sha "${GITHUB_SHA}" \
             --verified-source-sha "${GITHUB_SHA}" \
             --output .verification/holdout-aggregate-metrics.json
 
-      - name: Validate aggregate schema and leakage guard
+      - name: Validate aggregate schema shape
         if: steps.gate.outputs.configured == 'true'
         run: |
           python - <<'PY'
           import json
           from pathlib import Path
-          from scripts.run_formalpr_holdout import assert_aggregate_safe
-
+          from scripts.run_formalpr_holdout import validate_aggregate_schema
           payload = json.loads(Path(".verification/holdout-aggregate-metrics.json").read_text())
-          assert_aggregate_safe(payload)
-          assert payload["schema_version"] == "formalpr_holdout.aggregate_metrics.v1"
-          assert payload["leakage_guard"]["labels_emitted"] is False
-          assert payload["leakage_guard"]["case_ids_emitted"] is False
-          assert payload["leakage_guard"]["fail_closed"] is True
-          print("aggregate schema and leakage checks passed")
+          validate_aggregate_schema(payload)
+          print("aggregate schema checks passed; labels not present")
           PY
 
       - name: Upload aggregates only

benchmarks/formal_pr_bench/score_all_lanes.py

@@ -43,7 +43,9 @@ def evaluate_lane_case(case: dict) -> tuple[str, str, str | None]:
     if intent == "agent-cannot-disable-own-ci-gate":
         from ovk.adapters.opa import evaluate_self_protection
 
-        evidence = evaluate_self_protection(json.loads(Path(fixture).read_text(encoding="utf-8")), repo="bench/repo", head_sha="seed")
+        evidence = evaluate_self_protection(
+            json.loads(Path(fixture).read_text(encoding="utf-8")), repo="bench/repo", head_sha="seed"
+        )
         payload = evidence.model_dump(mode="json")
     elif intent in INTENT_TO_LANE:
         payload = _evaluate_lane_fixture(INTENT_TO_LANE[intent], fixture)
@@ -109,10 +111,7 @@ def run_benchmark(
     cases, case_set = load_cases(expanded=expanded, include_extended=include_extended)
     capabilities = CapabilityRegistry.from_directory(ROOT / "adapters").all()
     lane_evaluator: Callable[[dict[str, Any]], tuple[str, str, str | None]] = evaluate_lane_case
-    scores = [
-        score_case(case, capabilities=capabilities, lane_evaluator=lane_evaluator)
-        for case in cases
-    ]
+    scores = [score_case(case, capabilities=capabilities, lane_evaluator=lane_evaluator) for case in cases]
     leaderboard = build_leaderboard(
         scores,
         benchmark_name="FormalPR-Bench",

benchmarks/formal_pr_bench/score_authorization_obligation.py

@@ -32,8 +32,7 @@ def score_malformed_fixture(path: Path) -> bool:
     status_ok = evidence.backend_claims[0].status.value == "unknown"
     decision_ok = evidence.decision["merge_recommendation"] == "require_human_review"
     diagnostic_ok = any(
-        item.get("failure_mode") == "authorization_abstraction_invalid"
-        for item in evidence.counterexamples
+        item.get("failure_mode") == "authorization_abstraction_invalid" for item in evidence.counterexamples
     )
     return status_ok and decision_ok and diagnostic_ok
 

benchmarks/formal_pr_bench/scoring.py

@@ -113,9 +113,7 @@ def score_routing_case(case: dict[str, Any], *, elapsed_ms: float) -> DimensionS
 
 def score_quality_case(case: dict[str, Any], *, elapsed_ms: float) -> DimensionScore:
     """Score evidence honesty via the quality gate on a bundle fixture."""
-    bundle = EvidenceBundle.model_validate(
-        json.loads((ROOT / case["input_fixture"]).read_text(encoding="utf-8"))
-    )
+    bundle = EvidenceBundle.model_validate(json.loads((ROOT / case["input_fixture"]).read_text(encoding="utf-8")))
     report = build_evidence_quality_report(bundle)
     expected_pass = bool(case.get("expected_quality_passed", False))
     honest = report.passed == expected_pass
@@ -143,9 +141,7 @@ def score_repair_loop_case(case: dict[str, Any], *, elapsed_ms: float) -> Dimens
     recommendation = str(result.bundle.decision.get("merge_recommendation", "unknown"))
     merge_ok = recommendation == case["expected_merge_recommendation"]
     counterexamples = [
-        counterexample
-        for evidence in result.bundle.evidence
-        for counterexample in evidence.counterexamples
+        counterexample for evidence in result.bundle.evidence for counterexample in evidence.counterexamples
     ]
     hints = [repair_hint_for_counterexample(item) for item in counterexamples]
     expected_fix = case.get("expected_fix_class")
@@ -343,9 +339,7 @@ def _rate(values: list[bool | None]) -> float | None:
         "evidence_honesty": _rate([score.evidence_honest for score in scores]),
         "intent_recall": _rate([score.status_correct for score in scores if score.category == "intent_recall"]),
         "real_diff_recall": _rate([score.status_correct for score in scores if score.category == "real_diff"]),
-        "real_diff_intent_recall": _rate(
-            [score.status_correct for score in scores if score.category == "real_diff"]
-        ),
+        "real_diff_intent_recall": _rate([score.status_correct for score in scores if score.category == "real_diff"]),
         "by_category": {
             category: {
                 "cases_total": len(items),

docs/ATTRIBUTABLE_PUBLICATION.md

@@ -0,0 +1,51 @@
+# Attributable Publication Checklist (Sprint 10)
+
+Gate for publishing **`v1.3.0-rc.1`** and later promoting to **`v1.3.0`**.
+Authority: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) 18-condition gate.
+
+## Terminology
+
+| Field | Use |
+|---|---|
+| `benchmark_source_sha` | FormalPR-Bench / badge measurement identity |
+| `verified_source_sha` | Complete observed required-workflow set only |
+
+Never label a `[skip ci]` badge commit as verified. Never re-attribute `v1.2.1`
+Sigstore / consumer evidence to typed-control-plane commits.
+
+## Collect workflow evidence (when Actions are available)
+
+```bash
+python scripts/collect_workflow_evidence.py \
+  --sha <SOURCE_SHA> \
+  --output .verification/workflow-evidence-<SOURCE_SHA>.json
+```
+
+The collector records run IDs/URLs under `benchmark_source_sha` and leaves
+`verified_source_sha` unset until maintainers confirm the full required set.
+
+## Pre-tag checklist (`v1.3.0-rc.1`)
+
+- [ ] P0 trust PRs 1–9 landed on the tag source
+- [ ] Non-`[skip ci]` CI, native Tier 1, wheel smoke, Action dogfood, release preflight green
+- [ ] Expanded FormalPR-Bench recorded with `benchmark_source_sha`
+- [ ] Template conformance v2 matrix regenerated from semantic statuses
+- [ ] Both consumers dispatched on immutable rc.1 pin (or audited commit); evidence downloaded and verified
+- [ ] Label-separated holdout aggregates retained (predictions digest + eval workflow IDs)
+- [ ] Release artifacts signed; workflow IDs and digests recorded in [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md)
+
+## Promote to `v1.3.0`
+
+Only after:
+
+- [ ] All 18 completion-gate conditions hold
+- [ ] P0 closure (PRs 1–9) on the exact tag source
+- [ ] Consumer validation on the exact pin
+- [ ] Attributable holdout aggregates (predictions digest + eval)
+- [ ] Human pilot ledgers remain separate from automated fixtures
+- [ ] No re-attribution of `v1.2.1` Sigstore evidence to typed-control-plane commits
+
+## Blocked without external access
+
+Live GitHub Actions run URLs, consumer repo pin PRs, and private holdout evaluation require
+maintainer credentials outside this working tree.

docs/CONSUMER_VALIDATION_CHECKLIST.md

@@ -1,46 +1,98 @@
-# Consumer Validation Checklist
-
-Scaffolding and live pointers for independent consumer repositories validating OVK.
-Completing this checklist for one repo does **not** satisfy the multi-repo production
-exit criterion (30 human-adjudicated PRs per independent consumer).
-
-## Live independent consumers (v1.2.1)
-
-| Repository | Stack | Ledger |
-|---|---|---|
-| [fraware/ovk-consumer-fastapi-terraform](https://github.com/fraware/ovk-consumer-fastapi-terraform) | FastAPI + Terraform | [`pilot/ledger.json`](https://github.com/fraware/ovk-consumer-fastapi-terraform/blob/main/pilot/ledger.json) |
-| [fraware/ovk-consumer-express-actions](https://github.com/fraware/ovk-consumer-express-actions) | Express + GitHub Actions | [`pilot/ledger.json`](https://github.com/fraware/ovk-consumer-express-actions/blob/main/pilot/ledger.json) |
-
-Both pin `fraware/open-verification-kernel@v1.2.1` (never `uses: ./`). CI fails on pin drift via `scripts/assert_ovk_pin.py`.
-
-## Immutable pin requirements
-
-Consumers must pin an **immutable** OVK commit SHA or release tag.
-
-```yaml
-env:
-  OVK_PACKAGE_VERSION: "1.2.1"
-steps:
-  - uses: fraware/open-verification-kernel@v1.2.1
-```
-
-Forbidden:
-
-- `uses: ./`
-- `uses: fraware/open-verification-kernel@main`
-- floating refs without a tag or full commit SHA
-
-## Checklist (per consumer)
-
-- [x] Workflow copies from `docs/templates/consumer_validation.workflow.yml` (or equivalent) with an immutable pin.
-- [x] Automated scenario matrix covers program section 23.1 intents (see consumer README).
-- [x] Adjudication rows recorded in a pilot ledger conforming to `schemas/pilot.ledger.schema.json`.
-- [ ] Human adjudications reach 30 PRs (entries must not remain `automated_scenario` / `pending` only).
-- [ ] True cross-fork PR exercised and ledger-adjudicated (see consumer `docs/FORK_PR.md`).
-- [ ] Prefer PyPI `open-verification-kernel==1.2.1` once published; until then Release wheel + cosign verify-blob.
-
-## What this does not claim
-
-- Declaring two independent consumer repos with 30 adjudicated PRs complete
-- Vision completion or Production-stable package status
-- That FormalPR-Holdout results generalize to these consumers (holdout is a separate program)
+# Consumer Validation Checklist
+
+Scaffolding and live pointers for independent consumer repositories validating OVK.
+Completing this checklist for one repo does **not** satisfy the multi-repo production
+exit criterion (30 human-adjudicated PRs per independent consumer).
+
+## Live independent consumers
+
+| Repository | Stack | Current pin | Target pin (Sprint 9) |
+|---|---|---|---|
+| [fraware/ovk-consumer-fastapi-terraform](https://github.com/fraware/ovk-consumer-fastapi-terraform) | FastAPI + Terraform | `v1.2.1` | immutable `v1.3.0-rc.1` (or audited commit) |
+| [fraware/ovk-consumer-express-actions](https://github.com/fraware/ovk-consumer-express-actions) | Express + GitHub Actions | `v1.2.1` | immutable `v1.3.0-rc.1` (or audited commit) |
+
+`v1.2.1` validates the **pre-control-plane** signed release only. Typed control-plane
+commits must not inherit that consumer evidence. Both consumers use
+`scripts/assert_ovk_pin.py` to fail on pin drift.
+
+## Immutable pin requirements
+
+Consumers must pin an **immutable** OVK commit SHA or release tag.
+
+```yaml
+env:
+  OVK_PACKAGE_VERSION: "1.3.0rc1"   # after rc.1 cut; until then keep 1.2.1
+  OVK_ACTION_REF: "v1.3.0-rc.1"
+steps:
+  - uses: fraware/open-verification-kernel@v1.3.0-rc.1
+```
+
+In-repo template: [templates/consumer_validation.workflow.yml](templates/consumer_validation.workflow.yml).
+
+Forbidden:
+
+- `uses: ./`
+- `uses: fraware/open-verification-kernel@main`
+- floating refs without a tag or full commit SHA
+
+## Maintainer steps after `v1.3.0-rc.1` exists (do not push from this workspace alone)
+
+For each consumer repository:
+
+1. Open a pin PR that bumps Action `uses:` and `OVK_PACKAGE_VERSION` to the immutable rc.1 tag (or full SHA).
+2. Merge the pin PR (or push to a validation branch) so workflows can see the new pin.
+3. Dispatch validation:
+   ```bash
+   gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-fastapi-terraform
+   gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-express-actions
+   ```
+   (Use the exact workflow name as defined in each consumer.)
+4. Await conclusions; download evidence artifacts:
+   ```bash
+   gh run download <RUN_ID> --repo <consumer> -n <evidence-artifact-name> -D ./consumer-evidence/<consumer>/
+   ```
+5. Verify bundles with the OVK release verifier + cosign as applicable for the pin.
+6. Exercise a true cross-fork PR path (`docs/FORK_PR.md` in each consumer).
+7. Update the pilot ledger: keep `automated_scenario` rows distinct from human adjudications.
+
+Local clone prep (optional, no push):
+
+```bash
+git clone https://github.com/fraware/ovk-consumer-fastapi-terraform.git
+git clone https://github.com/fraware/ovk-consumer-express-actions.git
+# Edit workflow pins locally; do not git push until maintainers cut rc.1.
+```
+
+## Sprint 9 checklist (per consumer) — prepare in this repo; land in consumer repos
+
+In-repo preparation (this repository):
+
+- [x] Document rc.1 target pins and provenance correction (this checklist + R2 status)
+- [x] Keep human pilot ledgers separate from automated fixtures (see consumer `pilot/ledger.json` policy)
+- [x] Template workflow targets `v1.3.0-rc.1` (copy only after tag exists)
+- [ ] Cut attributable `v1.3.0-rc.1` tag on verified source (Sprint 10)
+
+In consumer repositories (requires write access — **blocked from this workspace alone**):
+
+- [ ] Bump Action pin from `v1.2.1` → `v1.3.0-rc.1` (or audited full SHA)
+- [ ] Bump `OVK_PACKAGE_VERSION` / wheel install scripts to match
+- [ ] Dispatch validation workflows; await conclusions
+- [ ] Download evidence bundles; verify with release verifier + cosign as applicable
+- [ ] Exercise true cross-fork PR path (`docs/FORK_PR.md`)
+- [ ] Update ledger: automated scenarios remain distinct from human adjudications
+
+## Checklist (per consumer) — ongoing
+
+- [x] Workflow copies from `docs/templates/consumer_validation.workflow.yml` (or equivalent) with an immutable pin.
+- [x] Automated scenario matrix covers program section 23.1 intents (see consumer README).
+- [x] Adjudication rows recorded in a pilot ledger conforming to `schemas/pilot.ledger.schema.json`.
+- [ ] Human adjudications reach 30 PRs (entries must not remain `automated_scenario` / `pending` only).
+- [ ] True cross-fork PR exercised and ledger-adjudicated (see consumer `docs/FORK_PR.md`).
+- [ ] Prefer PyPI once published; until then Release wheel + cosign verify-blob at the **current** pin.
+
+## What this does not claim
+
+- Declaring two independent consumer repos with 30 adjudicated PRs complete
+- Vision completion or Production-stable package status
+- That FormalPR-Holdout results generalize to these consumers (holdout is a separate program)
+- That `v1.2.1` consumer green runs validate typed-control-plane `main`