diff --git a/.github/workflows/holdout-eval.yml b/.github/workflows/holdout-eval.yml index 2785cf8..c7159fe 100644 --- a/.github/workflows/holdout-eval.yml +++ b/.github/workflows/holdout-eval.yml @@ -4,6 +4,7 @@ name: FormalPR-Holdout eval # Requires repository secret HOLDOUT_DOWNLOAD_TOKEN with read access to # fraware/FormalPR-Holdout private release assets. The independently recorded # asset SHA-256 is required so a mutable/replaced release asset cannot execute. +# Prefer workflow input asset_sha256; otherwise repository variable HOLDOUT_ASSET_SHA256. on: workflow_dispatch: inputs: @@ -12,9 +13,10 @@ on: required: true default: v0.1.0-synthetic type: string - holdout_asset_sha256: - description: Independently recorded SHA-256 of the frozen release asset - required: true + asset_sha256: + description: Immutable SHA-256 of the release asset (overrides repo variable) + required: false + default: "" type: string predictions_artifact: description: Optional name of a workflow artifact containing predictions.json @@ -33,13 +35,19 @@ jobs: id: gate env: HOLDOUT_DOWNLOAD_TOKEN: ${{ secrets.HOLDOUT_DOWNLOAD_TOKEN }} + HOLDOUT_ASSET_SHA256: ${{ inputs.asset_sha256 || vars.HOLDOUT_ASSET_SHA256 }} run: | if [ -z "$HOLDOUT_DOWNLOAD_TOKEN" ]; then echo "configured=false" >> "$GITHUB_OUTPUT" echo "FormalPR-Holdout eval skipped: HOLDOUT_DOWNLOAD_TOKEN not configured." echo "See docs/FORMALPR_HOLDOUT_GOVERNANCE.md" + elif [ -z "$HOLDOUT_ASSET_SHA256" ]; then + echo "configured=false" >> "$GITHUB_OUTPUT" + echo "FormalPR-Holdout eval skipped: HOLDOUT_ASSET_SHA256 not configured." + echo "Provide workflow input asset_sha256 or repository variable HOLDOUT_ASSET_SHA256." else echo "configured=true" >> "$GITHUB_OUTPUT" + echo "asset_sha256=$HOLDOUT_ASSET_SHA256" >> "$GITHUB_OUTPUT" fi - uses: actions/checkout@v4 @@ -50,12 +58,6 @@ jobs: with: python-version: "3.11" - - name: Install OVK runner dependencies - if: steps.gate.outputs.configured == 'true' - run: | - python -m pip install --upgrade pip - pip install -e . - - name: Require predictions file if: steps.gate.outputs.configured == 'true' id: preds @@ -87,35 +89,61 @@ jobs: fi test -f .verification/holdout-predictions.json - - name: Run holdout aggregate eval (fail closed) + - name: Download holdout asset (token only here) if: steps.gate.outputs.configured == 'true' + id: download env: HOLDOUT_DOWNLOAD_TOKEN: ${{ secrets.HOLDOUT_DOWNLOAD_TOKEN }} - HOLDOUT_ASSET_SHA256: ${{ inputs.holdout_asset_sha256 }} run: | + python - <<'PY' + import os + from pathlib import Path + from scripts.run_formalpr_holdout import download_release_asset, verify_asset_sha256 + + tag = "${{ inputs.holdout_tag }}" + asset_name = f"FormalPR-Holdout-{tag}.tar.gz" + dest = Path(".verification") / asset_name + dest.parent.mkdir(parents=True, exist_ok=True) + download_release_asset( + repo="fraware/FormalPR-Holdout", + tag=tag, + asset_name=asset_name, + dest=dest, + token=os.environ["HOLDOUT_DOWNLOAD_TOKEN"], + ) + verify_asset_sha256(dest, "${{ steps.gate.outputs.asset_sha256 }}") + print(f"downloaded and verified {dest}") + PY + echo "artifact=.verification/FormalPR-Holdout-${{ inputs.holdout_tag }}.tar.gz" >> "$GITHUB_OUTPUT" + + - name: Run holdout aggregate eval (fail closed, no tokens) + if: steps.gate.outputs.configured == 'true' + env: + # Explicitly unset download tokens for the evaluator process. + HOLDOUT_DOWNLOAD_TOKEN: "" + GITHUB_TOKEN: "" + GH_TOKEN: "" + run: | + unset HOLDOUT_DOWNLOAD_TOKEN GITHUB_TOKEN GH_TOKEN || true python scripts/run_formalpr_holdout.py \ + --artifact "${{ steps.download.outputs.artifact }}" \ + --asset-sha256 "${{ steps.gate.outputs.asset_sha256 }}" \ --tag "${{ inputs.holdout_tag }}" \ - --asset-sha256 "${HOLDOUT_ASSET_SHA256}" \ --predictions .verification/holdout-predictions.json \ --ovk-commit-sha "${GITHUB_SHA}" \ --verified-source-sha "${GITHUB_SHA}" \ --output .verification/holdout-aggregate-metrics.json - - name: Validate aggregate schema and leakage guard + - name: Validate aggregate schema shape if: steps.gate.outputs.configured == 'true' run: | python - <<'PY' import json from pathlib import Path - from scripts.run_formalpr_holdout import assert_aggregate_safe - + from scripts.run_formalpr_holdout import validate_aggregate_schema payload = json.loads(Path(".verification/holdout-aggregate-metrics.json").read_text()) - assert_aggregate_safe(payload) - assert payload["schema_version"] == "formalpr_holdout.aggregate_metrics.v1" - assert payload["leakage_guard"]["labels_emitted"] is False - assert payload["leakage_guard"]["case_ids_emitted"] is False - assert payload["leakage_guard"]["fail_closed"] is True - print("aggregate schema and leakage checks passed") + validate_aggregate_schema(payload) + print("aggregate schema checks passed; labels not present") PY - name: Upload aggregates only diff --git a/benchmarks/formal_pr_bench/score_all_lanes.py b/benchmarks/formal_pr_bench/score_all_lanes.py index 279df5f..bddc8b8 100644 --- a/benchmarks/formal_pr_bench/score_all_lanes.py +++ b/benchmarks/formal_pr_bench/score_all_lanes.py @@ -43,7 +43,9 @@ def evaluate_lane_case(case: dict) -> tuple[str, str, str | None]: if intent == "agent-cannot-disable-own-ci-gate": from ovk.adapters.opa import evaluate_self_protection - evidence = evaluate_self_protection(json.loads(Path(fixture).read_text(encoding="utf-8")), repo="bench/repo", head_sha="seed") + evidence = evaluate_self_protection( + json.loads(Path(fixture).read_text(encoding="utf-8")), repo="bench/repo", head_sha="seed" + ) payload = evidence.model_dump(mode="json") elif intent in INTENT_TO_LANE: payload = _evaluate_lane_fixture(INTENT_TO_LANE[intent], fixture) @@ -109,10 +111,7 @@ def run_benchmark( cases, case_set = load_cases(expanded=expanded, include_extended=include_extended) capabilities = CapabilityRegistry.from_directory(ROOT / "adapters").all() lane_evaluator: Callable[[dict[str, Any]], tuple[str, str, str | None]] = evaluate_lane_case - scores = [ - score_case(case, capabilities=capabilities, lane_evaluator=lane_evaluator) - for case in cases - ] + scores = [score_case(case, capabilities=capabilities, lane_evaluator=lane_evaluator) for case in cases] leaderboard = build_leaderboard( scores, benchmark_name="FormalPR-Bench", diff --git a/benchmarks/formal_pr_bench/score_authorization_obligation.py b/benchmarks/formal_pr_bench/score_authorization_obligation.py index 74f7a04..2480c7b 100644 --- a/benchmarks/formal_pr_bench/score_authorization_obligation.py +++ b/benchmarks/formal_pr_bench/score_authorization_obligation.py @@ -32,8 +32,7 @@ def score_malformed_fixture(path: Path) -> bool: status_ok = evidence.backend_claims[0].status.value == "unknown" decision_ok = evidence.decision["merge_recommendation"] == "require_human_review" diagnostic_ok = any( - item.get("failure_mode") == "authorization_abstraction_invalid" - for item in evidence.counterexamples + item.get("failure_mode") == "authorization_abstraction_invalid" for item in evidence.counterexamples ) return status_ok and decision_ok and diagnostic_ok diff --git a/benchmarks/formal_pr_bench/scoring.py b/benchmarks/formal_pr_bench/scoring.py index 52d6439..5317dff 100644 --- a/benchmarks/formal_pr_bench/scoring.py +++ b/benchmarks/formal_pr_bench/scoring.py @@ -113,9 +113,7 @@ def score_routing_case(case: dict[str, Any], *, elapsed_ms: float) -> DimensionS def score_quality_case(case: dict[str, Any], *, elapsed_ms: float) -> DimensionScore: """Score evidence honesty via the quality gate on a bundle fixture.""" - bundle = EvidenceBundle.model_validate( - json.loads((ROOT / case["input_fixture"]).read_text(encoding="utf-8")) - ) + bundle = EvidenceBundle.model_validate(json.loads((ROOT / case["input_fixture"]).read_text(encoding="utf-8"))) report = build_evidence_quality_report(bundle) expected_pass = bool(case.get("expected_quality_passed", False)) honest = report.passed == expected_pass @@ -143,9 +141,7 @@ def score_repair_loop_case(case: dict[str, Any], *, elapsed_ms: float) -> Dimens recommendation = str(result.bundle.decision.get("merge_recommendation", "unknown")) merge_ok = recommendation == case["expected_merge_recommendation"] counterexamples = [ - counterexample - for evidence in result.bundle.evidence - for counterexample in evidence.counterexamples + counterexample for evidence in result.bundle.evidence for counterexample in evidence.counterexamples ] hints = [repair_hint_for_counterexample(item) for item in counterexamples] expected_fix = case.get("expected_fix_class") @@ -343,9 +339,7 @@ def _rate(values: list[bool | None]) -> float | None: "evidence_honesty": _rate([score.evidence_honest for score in scores]), "intent_recall": _rate([score.status_correct for score in scores if score.category == "intent_recall"]), "real_diff_recall": _rate([score.status_correct for score in scores if score.category == "real_diff"]), - "real_diff_intent_recall": _rate( - [score.status_correct for score in scores if score.category == "real_diff"] - ), + "real_diff_intent_recall": _rate([score.status_correct for score in scores if score.category == "real_diff"]), "by_category": { category: { "cases_total": len(items), diff --git a/docs/ATTRIBUTABLE_PUBLICATION.md b/docs/ATTRIBUTABLE_PUBLICATION.md new file mode 100644 index 0000000..83affe7 --- /dev/null +++ b/docs/ATTRIBUTABLE_PUBLICATION.md @@ -0,0 +1,51 @@ +# Attributable Publication Checklist (Sprint 10) + +Gate for publishing **`v1.3.0-rc.1`** and later promoting to **`v1.3.0`**. +Authority: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) 18-condition gate. + +## Terminology + +| Field | Use | +|---|---| +| `benchmark_source_sha` | FormalPR-Bench / badge measurement identity | +| `verified_source_sha` | Complete observed required-workflow set only | + +Never label a `[skip ci]` badge commit as verified. Never re-attribute `v1.2.1` +Sigstore / consumer evidence to typed-control-plane commits. + +## Collect workflow evidence (when Actions are available) + +```bash +python scripts/collect_workflow_evidence.py \ + --sha \ + --output .verification/workflow-evidence-.json +``` + +The collector records run IDs/URLs under `benchmark_source_sha` and leaves +`verified_source_sha` unset until maintainers confirm the full required set. + +## Pre-tag checklist (`v1.3.0-rc.1`) + +- [ ] P0 trust PRs 1–9 landed on the tag source +- [ ] Non-`[skip ci]` CI, native Tier 1, wheel smoke, Action dogfood, release preflight green +- [ ] Expanded FormalPR-Bench recorded with `benchmark_source_sha` +- [ ] Template conformance v2 matrix regenerated from semantic statuses +- [ ] Both consumers dispatched on immutable rc.1 pin (or audited commit); evidence downloaded and verified +- [ ] Label-separated holdout aggregates retained (predictions digest + eval workflow IDs) +- [ ] Release artifacts signed; workflow IDs and digests recorded in [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) + +## Promote to `v1.3.0` + +Only after: + +- [ ] All 18 completion-gate conditions hold +- [ ] P0 closure (PRs 1–9) on the exact tag source +- [ ] Consumer validation on the exact pin +- [ ] Attributable holdout aggregates (predictions digest + eval) +- [ ] Human pilot ledgers remain separate from automated fixtures +- [ ] No re-attribution of `v1.2.1` Sigstore evidence to typed-control-plane commits + +## Blocked without external access + +Live GitHub Actions run URLs, consumer repo pin PRs, and private holdout evaluation require +maintainer credentials outside this working tree. diff --git a/docs/CONSUMER_VALIDATION_CHECKLIST.md b/docs/CONSUMER_VALIDATION_CHECKLIST.md index a046634..da4b213 100644 --- a/docs/CONSUMER_VALIDATION_CHECKLIST.md +++ b/docs/CONSUMER_VALIDATION_CHECKLIST.md @@ -1,46 +1,98 @@ -# Consumer Validation Checklist - -Scaffolding and live pointers for independent consumer repositories validating OVK. -Completing this checklist for one repo does **not** satisfy the multi-repo production -exit criterion (30 human-adjudicated PRs per independent consumer). - -## Live independent consumers (v1.2.1) - -| Repository | Stack | Ledger | -|---|---|---| -| [fraware/ovk-consumer-fastapi-terraform](https://github.com/fraware/ovk-consumer-fastapi-terraform) | FastAPI + Terraform | [`pilot/ledger.json`](https://github.com/fraware/ovk-consumer-fastapi-terraform/blob/main/pilot/ledger.json) | -| [fraware/ovk-consumer-express-actions](https://github.com/fraware/ovk-consumer-express-actions) | Express + GitHub Actions | [`pilot/ledger.json`](https://github.com/fraware/ovk-consumer-express-actions/blob/main/pilot/ledger.json) | - -Both pin `fraware/open-verification-kernel@v1.2.1` (never `uses: ./`). CI fails on pin drift via `scripts/assert_ovk_pin.py`. - -## Immutable pin requirements - -Consumers must pin an **immutable** OVK commit SHA or release tag. - -```yaml -env: - OVK_PACKAGE_VERSION: "1.2.1" -steps: - - uses: fraware/open-verification-kernel@v1.2.1 -``` - -Forbidden: - -- `uses: ./` -- `uses: fraware/open-verification-kernel@main` -- floating refs without a tag or full commit SHA - -## Checklist (per consumer) - -- [x] Workflow copies from `docs/templates/consumer_validation.workflow.yml` (or equivalent) with an immutable pin. -- [x] Automated scenario matrix covers program section 23.1 intents (see consumer README). -- [x] Adjudication rows recorded in a pilot ledger conforming to `schemas/pilot.ledger.schema.json`. -- [ ] Human adjudications reach 30 PRs (entries must not remain `automated_scenario` / `pending` only). -- [ ] True cross-fork PR exercised and ledger-adjudicated (see consumer `docs/FORK_PR.md`). -- [ ] Prefer PyPI `open-verification-kernel==1.2.1` once published; until then Release wheel + cosign verify-blob. - -## What this does not claim - -- Declaring two independent consumer repos with 30 adjudicated PRs complete -- Vision completion or Production-stable package status -- That FormalPR-Holdout results generalize to these consumers (holdout is a separate program) +# Consumer Validation Checklist + +Scaffolding and live pointers for independent consumer repositories validating OVK. +Completing this checklist for one repo does **not** satisfy the multi-repo production +exit criterion (30 human-adjudicated PRs per independent consumer). + +## Live independent consumers + +| Repository | Stack | Current pin | Target pin (Sprint 9) | +|---|---|---|---| +| [fraware/ovk-consumer-fastapi-terraform](https://github.com/fraware/ovk-consumer-fastapi-terraform) | FastAPI + Terraform | `v1.2.1` | immutable `v1.3.0-rc.1` (or audited commit) | +| [fraware/ovk-consumer-express-actions](https://github.com/fraware/ovk-consumer-express-actions) | Express + GitHub Actions | `v1.2.1` | immutable `v1.3.0-rc.1` (or audited commit) | + +`v1.2.1` validates the **pre-control-plane** signed release only. Typed control-plane +commits must not inherit that consumer evidence. Both consumers use +`scripts/assert_ovk_pin.py` to fail on pin drift. + +## Immutable pin requirements + +Consumers must pin an **immutable** OVK commit SHA or release tag. + +```yaml +env: + OVK_PACKAGE_VERSION: "1.3.0rc1" # after rc.1 cut; until then keep 1.2.1 + OVK_ACTION_REF: "v1.3.0-rc.1" +steps: + - uses: fraware/open-verification-kernel@v1.3.0-rc.1 +``` + +In-repo template: [templates/consumer_validation.workflow.yml](templates/consumer_validation.workflow.yml). + +Forbidden: + +- `uses: ./` +- `uses: fraware/open-verification-kernel@main` +- floating refs without a tag or full commit SHA + +## Maintainer steps after `v1.3.0-rc.1` exists (do not push from this workspace alone) + +For each consumer repository: + +1. Open a pin PR that bumps Action `uses:` and `OVK_PACKAGE_VERSION` to the immutable rc.1 tag (or full SHA). +2. Merge the pin PR (or push to a validation branch) so workflows can see the new pin. +3. Dispatch validation: + ```bash + gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-fastapi-terraform + gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-express-actions + ``` + (Use the exact workflow name as defined in each consumer.) +4. Await conclusions; download evidence artifacts: + ```bash + gh run download --repo -n -D ./consumer-evidence// + ``` +5. Verify bundles with the OVK release verifier + cosign as applicable for the pin. +6. Exercise a true cross-fork PR path (`docs/FORK_PR.md` in each consumer). +7. Update the pilot ledger: keep `automated_scenario` rows distinct from human adjudications. + +Local clone prep (optional, no push): + +```bash +git clone https://github.com/fraware/ovk-consumer-fastapi-terraform.git +git clone https://github.com/fraware/ovk-consumer-express-actions.git +# Edit workflow pins locally; do not git push until maintainers cut rc.1. +``` + +## Sprint 9 checklist (per consumer) — prepare in this repo; land in consumer repos + +In-repo preparation (this repository): + +- [x] Document rc.1 target pins and provenance correction (this checklist + R2 status) +- [x] Keep human pilot ledgers separate from automated fixtures (see consumer `pilot/ledger.json` policy) +- [x] Template workflow targets `v1.3.0-rc.1` (copy only after tag exists) +- [ ] Cut attributable `v1.3.0-rc.1` tag on verified source (Sprint 10) + +In consumer repositories (requires write access — **blocked from this workspace alone**): + +- [ ] Bump Action pin from `v1.2.1` → `v1.3.0-rc.1` (or audited full SHA) +- [ ] Bump `OVK_PACKAGE_VERSION` / wheel install scripts to match +- [ ] Dispatch validation workflows; await conclusions +- [ ] Download evidence bundles; verify with release verifier + cosign as applicable +- [ ] Exercise true cross-fork PR path (`docs/FORK_PR.md`) +- [ ] Update ledger: automated scenarios remain distinct from human adjudications + +## Checklist (per consumer) — ongoing + +- [x] Workflow copies from `docs/templates/consumer_validation.workflow.yml` (or equivalent) with an immutable pin. +- [x] Automated scenario matrix covers program section 23.1 intents (see consumer README). +- [x] Adjudication rows recorded in a pilot ledger conforming to `schemas/pilot.ledger.schema.json`. +- [ ] Human adjudications reach 30 PRs (entries must not remain `automated_scenario` / `pending` only). +- [ ] True cross-fork PR exercised and ledger-adjudicated (see consumer `docs/FORK_PR.md`). +- [ ] Prefer PyPI once published; until then Release wheel + cosign verify-blob at the **current** pin. + +## What this does not claim + +- Declaring two independent consumer repos with 30 adjudicated PRs complete +- Vision completion or Production-stable package status +- That FormalPR-Holdout results generalize to these consumers (holdout is a separate program) +- That `v1.2.1` consumer green runs validate typed-control-plane `main` diff --git a/docs/CURRENT_RELEASE_STATUS.md b/docs/CURRENT_RELEASE_STATUS.md index bb55c66..71efc75 100644 --- a/docs/CURRENT_RELEASE_STATUS.md +++ b/docs/CURRENT_RELEASE_STATUS.md @@ -1,132 +1,126 @@ -# OVK Release Status - -Living release and adoption dashboard for Open Verification Kernel. - -**Last updated:** 2026-07-23 - -## Release judgment - -The currently published and signed release is `v1.2.1` at commit: - -`a27d5720f4350c00bca34f71d991c31f5a2f38c7` - -Release workflow run `30010876652` successfully completed release verification, package build, isolated wheel smoke, and keyless Sigstore signing for that tag. PyPI publication was skipped. - -Current `main` is a post-v1.2.1 development line. It adds the typed backend control plane, enforced lane adapters, source compilers, evidence v2, template conformance, holdout infrastructure, consumer validation scaffolding, and later audit fixes. The v1.2.1 run does not validate these post-tag changes. - -**Current-main judgment:** advanced release candidate for a future `v1.3.0-rc.1`, pending P0 trust fixes and current-source CI. - -Authoritative current audit: - -- [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) - -Standalone engineer instructions: - -- [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md) - -## At a glance - -| Signal | Current state | -|---|---| -| **Published/signed release** | `v1.2.1` at `a27d572…`; valid evidence for the previous release only | -| **Current development line** | Post-v1.2.1 control-plane architecture; requires a new release-candidate cycle | -| **FormalPR-Bench** | Repository snapshot reports 130/130 curated regression cases; internal conformance, not external accuracy | -| **Production lanes** | Self-protection, authorization, infrastructure, CI secrets, deployment | -| **Backend routing** | Typed routing controls execution inside explicitly enforced lane paths | -| **Default routing mode** | Shadow; legacy evidence remains authoritative unless lanes are explicitly enforced | -| **Native semantic paths** | OPA and Z3; CBMC supports bounded explicit/template harness paths | -| **Deterministic adapters** | Five lane implementations plus external contract adapters | -| **Current-source CI** | Must be rerun after the latest engineer and audit changes | -| **Independent consumers** | Two public consumer repositories exist, but they currently pin `v1.2.1`, not current `main` | -| **Holdout** | Evaluator plumbing exists; prediction/evaluation separation and current-wheel evaluation remain incomplete | -| **Package status** | Beta | - -OVK is not complete formal verification of arbitrary code. It provides conservative verification evidence for bounded, explicitly modeled risk profiles. - -## What is now achieved - -- typed backend-neutral obligations; -- typed capability assessment and routing decisions; -- registry-controlled selected backend execution; -- five lane-specific enforced paths; -- fail-dominant aggregation; -- evidence v2 generation; -- routing- and environment-bound cache keys; -- source compiler packages; -- template catalog separation; -- manifests, provenance, attestations, HMAC, and Sigstore support; -- automatic GitHub Action PR-diff collection; -- two independent consumer repositories and immutable-pin checks. - -## Current P0 gaps - -1. Execution-attempt identity includes nondeterministic duration data. -2. Cache hits do not preserve the original execution attempt and can reconstruct native provenance from current tool availability. -3. Compiler/backend guarantee mismatch is silently rewritten instead of rejected. -4. Required-primary selection does not enforce `coverage_requirements_met`. -5. Fallback acceptance is not constrained to configured backend, guarantee, and failure cause. -6. Self-protection metadata is trusted by default unless policy explicitly disables trust. -7. Deterministic in-process adapters cannot be hard-cancelled. -8. Kernel inference and enforced execution still use separate routing paths. -9. Evidence v2 schema and invariants do not fully recompute and cross-bind routing, attempts, aggregate decisions, and materials. -10. Lane-specific infrastructure policy is not fully compiled into enforced execution semantics. -11. Source compilers remain advisory/profile-limited despite some `strict_eligible` labels. -12. Template strict eligibility is based primarily on executable-link presence. -13. Consumer repositories validate the previous release tag rather than current control-plane source. -14. Current source lacks attributable release-candidate CI, native-backend, consumer, and holdout evidence. - -## Direct audit fixes now on current main - -- CI-secrets material byte size now binds canonical serialized bytes. -- Backend subprocess workers inherit only a minimal allowlisted environment. -- Unknown ambient credentials are excluded from backend workers. -- Non-positive backend timeout prevents execution. -- Remote FormalPR-Holdout assets require an independently supplied SHA-256. -- Holdout archive extraction rejects traversal, links, devices, and special files. -- Downloaded holdout evaluators run without GitHub or holdout tokens. -- Holdout aggregates receive full JSON-schema and leakage-guard validation. - -These fixes require a fresh current-source CI run. - -## Adoption readiness - -| Mode | Recommendation | Conditions | -|---|---|---| -| **Local/demo** | Appropriate after current-source CI | Inspect assumptions, limits, and profile coverage | -| **Shadow Action** | Appropriate after current-source CI | Compare typed and legacy results; retain disagreements | -| **Advisory enforced lane** | Controlled pilots only | Explicit lane policy, trusted materials, coverage review | -| **Strict required check** | Repository/profile-specific | P0 fixes, calibrated source profile, protected policy and metadata | -| **Production-stable general enforcement** | Not yet | Current-source release evidence, semantic profiles, holdout, consumer pilots | - -## Release path - -The next release should use a new release-candidate version such as: - -`v1.3.0-rc.1` - -Before that tag: - -- [ ] close execution identity and cache provenance defects; -- [ ] enforce coverage and guarantee contracts; -- [ ] implement constrained fallback semantics; -- [ ] default self-protection metadata to untrusted; -- [ ] unify kernel routing; -- [ ] hard-isolate authoritative adapters; -- [ ] strengthen evidence and cross-artifact material binding; -- [ ] run current-source general and native CI; -- [ ] update both consumer repositories to the immutable release-candidate pin; -- [ ] dispatch and verify consumer scenarios; -- [ ] generate label-separated holdout predictions from the exact release-candidate artifact; -- [ ] retain exact source SHA, workflow IDs, wheel digest, Action artifacts, and signing bundles. - -## Related documents - -| Document | Purpose | -|---|---| -| [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | Fresh independent code, artifact, release, and vision audit | -| [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md) | Standalone implementation instructions and acceptance gates | -| [POST_MERGE_DEEP_AUDIT_2026-07-23.md](POST_MERGE_DEEP_AUDIT_2026-07-23.md) | Earlier post-merge audit, retained for history | -| [BACKENDS.md](BACKENDS.md) | Backend execution maturity | -| [RELEASE.md](RELEASE.md) | Release procedure | -| [FORMALPR_HOLDOUT_GOVERNANCE.md](FORMALPR_HOLDOUT_GOVERNANCE.md) | Holdout governance | -| [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) | Consumer validation checklist | +# OVK Release Status + +Living adoption dashboard for Open Verification Kernel. + +**Last updated:** 2026-07-23 + +**Release judgment:** **`v1.3.0-rc.1` candidate**. The typed backend control plane post-dates signed `v1.2.1` (`a27d5720f4350c00bca34f71d991c31f5a2f38c7`). Default product path remains shadow/legacy-authoritative; enforced routing is lane-policy opt-in until P0 trust closure. Do not treat current `main` as a re-validation of signed `v1.2.1`. + +Authoritative audit: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Engineering program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). Historical: [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) (superseded for day-to-day status). + +## At a glance + +| Signal | Current state | +|---|---| +| **Package version** | Working tree targets future `v1.3.0-rc.1`; signed immutable tag remains `v1.2.1` only for that tag’s commit | +| **FormalPR-Bench** | Internal curated regression; report `benchmark_source_sha` separately from `verified_source_sha` | +| **Check types** | Five bounded production lanes: self-protection, authorization, infrastructure, CI secrets, deployment | +| **Backend execution** | Typed `BackendControlPlane` + `route_obligation`; five policy-selectable enforced lanes via `adapter_runtime` | +| **Routing** | Enforced under lane policy; default path still shadow/legacy-authoritative until P0 closure | +| **Unit and workflow tests** | Local Sprint 0 baseline recorded below; live GitHub Actions workflow IDs still pending | +| **Package portability** | Local wheel-outside-checkout import smoke passed on working tree (package metadata still `1.2.1` until rc.1 cut) | +| **GitHub Action** | Consumers still live-pin `v1.2.1`; local consumer clones prepared for `v1.3.0-rc.1` (not pushed) | +| **External validation** | In-repository dogfooding + consumer scaffolding; independent pilots incomplete | +| **Sigstore** | Immutable-tag E2E closed for `v1.2.1` only — not attributable to typed control-plane commits | + +OVK is not complete formal verification of arbitrary code. It provides explainable, conservative checks for a bounded set of high-risk changes and emits explicit unknown and human-review outcomes. + +## Source SHA terminology + +| Field | Meaning | When to set | +|---|---|---| +| `benchmark_source_sha` | Commit whose FormalPR-Bench (or badge) artifacts were measured | Any bench/badge run | +| `verified_source_sha` | Commit with a **complete observed required-workflow set** | Only after Sprint 0 / release gates attach live workflow IDs | + +Badge-only or `[skip ci]` commits must set `benchmark_source_sha` and must **not** be labeled `verified_source_sha`. + +## Local Sprint 0 baseline + +Local evidence only. Distinguishes from GitHub Actions workflow IDs (still pending). Working tree HEAD at measurement time: `4b48ab245193e177a6d95e8557332334a9bd2883` (badge `[skip ci]` tip — treat as `benchmark_source_sha`, not verified). + +| Gate | Command | Exit | Timestamp (local) | +|---|---|---|---| +| Focused R2 + enforcement pytest | `python -m pytest tests/test_source_profile_hardening.py tests/test_template_conformance.py tests/test_verified_source.py tests/test_bench_badge.py tests/test_formalpr_holdout_runner.py tests/test_execution_models.py tests/test_cache_worker_control_plane.py tests/test_adapter_isolation_r2_pr8.py tests/test_evidence_v3_r2_pr9.py tests/test_authorization_enforcement.py tests/test_adversarial_control_plane.py tests/test_source_profiles.py -q` | **0** (111 passed) | 2026-07-23T23:41:09-07:00 → 23:42:03 | +| Broader compiler/cache suite | `python -m pytest tests/test_authorization_compilers.py tests/test_infrastructure_compilers.py tests/test_github_actions_trust.py tests/test_remaining_lane_enforcement.py tests/test_self_protection_enforcement.py tests/test_verification_cache.py tests/test_result_cache_semantics.py -q` | **0** (54 passed) | 2026-07-23T23:35:44-07:00 | +| Sprint 6–8 regression | `python -m pytest tests/test_source_profile_hardening.py tests/test_template_conformance.py tests/test_formalpr_holdout_runner.py tests/test_source_profiles.py tests/test_authorization_compilers.py tests/test_infrastructure_compilers.py -q` | **0** (46 passed) | 2026-07-23T23:40:48-07:00 | +| Release preflight (`PYTHONPATH=.`) | `python scripts/release_preflight.py` | **0** | 2026-07-23T23:44:17-07:00 → 23:45:10 | +| Template validation | `python scripts/validate_templates.py` | **0** | 2026-07-23T23:41:09-07:00 | +| Template conformance v2 regenerate | `python scripts/build_template_conformance.py` | **0** (`source_profile_strict_eligible=3`, `executable_advisory=2`, `catalog_only=95`) | 2026-07-23T23:40:57-07:00 | +| Local release smoke | `python scripts/smoke_release_local.py` | **0** | 2026-07-23T23:41:09-07:00 | +| Wheel build + outside-checkout import | `python -m build --wheel` then `pip install … -t $TEMP/ovk-outside-import` and `import ovk` | **0** (`verified_source_sha` correctly `None` outside attested env) | 2026-07-23T23:46:01-07:00 → 23:46:29 | +| Workflow ID collector | `python scripts/collect_workflow_evidence.py --sha --output .verification/workflow-evidence-local.json` | **0** (0 runs on `[skip ci]` tip; `verified_source_sha` left unset) | 2026-07-23T23:46:29-07:00 | + +### Still pending (live GitHub Actions / secrets) + +| Gate | Status | Evidence | +|---|---|---| +| General CI / unit+gates on non-`[skip ci]` SHA | Pending live run | Record run URL when available | +| Native Tier 1 | Pending | — | +| Action dogfood | Pending | — | +| Expanded FormalPR-Bench on release SHA | Pending | Use `benchmark_source_sha` | +| Adversarial release-bundle in Actions | Pending | Local `verify_release_bundle.py` entrypoint present | +| Label-separated holdout live eval | Pending | Needs `HOLDOUT_DOWNLOAD_TOKEN` + `HOLDOUT_ASSET_SHA256` | +| Consumer remotes on rc.1 | Pending push | Local clones prepared under `%TEMP%\ovk-consumer-prep\` (not pushed) | + +## Adoption readiness + +| Mode | Current recommendation | Conditions | +|---|---|---| +| **Local/demo** | Appropriate after current local/CI green | Use shipped examples and inspect assumptions and limits | +| **Advisory Action** | Appropriate for pilots on pinned tags | Prefer `v1.2.1` until rc.1 is attributable; collect FPs/unknowns | +| **Strict required check** | Repository-specific only | Calibrate on real diffs; trusted abstraction sources; protected policy metadata | +| **Production-stable general enforcement** | Not yet | P0 code (PRs 1–9) in working tree; still needs consumers on rc.1, attributable holdout, and Sprint 0 live gates | + +Suggested rollout: local validation → advisory artifacts → advisory check run/comment → calibrated strict lane → protected required check. + +## P0 trust defects (R2 PRs 1–9) — working-tree status + +Code for PRs 1–9 is present in this working tree (attempt identity excludes `duration_ms`; `ovk.cache.v3` / `CachedBackendExecution`; coverage/guarantee fail-closed; fallback v2 blocking terminations; `metadata_trusted` default false; authoritative routing pipeline; worker isolation; `ovk.evidence.v3` material-set binding). Historical defect inventory: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). + +**Still open for attributable release (external / Sprint 0–10):** live non-`[skip ci]` workflow IDs, consumer repo pins on immutable rc.1, label-separated holdout aggregates, and signed publication gates — see checklists below and [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). + +## Sprint 6–10 working-tree progress + +| Sprint | Local status | +|---|---| +| 6 Source profiles | FastAPI AST compiler; Terraform recursive modules; K8s controller reachability; Actions permissions-flow prover; deployment trusted-profile gate | +| 7 Template conformance v2 | Statuses derived from executed profile evidence (`source_profile_strict_eligible=3`; no `externally_calibrated_strict` from local gen) | +| 8 Holdout separation | `digest_holdout_predictions.py` + label-free guards; eval path token-stripped | +| 9 Consumers | In-repo template + checklist for rc.1; local clones pin-prepped (no push) | +| 10 Publication | [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md) + `scripts/collect_workflow_evidence.py` | + +## Maintainer release gates + +Before tagging or publishing **`v1.3.0-rc.1`**: + +- [ ] run all CI and native Tier 1 jobs on a non-`[skip ci]` source commit; +- [ ] confirm wheel smoke from a directory outside the checkout; +- [ ] confirm automatic-diff composite Action dogfood; +- [ ] confirm package version matches the release tag; +- [ ] run full expanded FormalPR-Bench and release preflight; +- [ ] validate a complete release bundle, including evidence-quality semantics; +- [ ] exercise HMAC signing and identity-bound Sigstore signing according to release policy; +- [ ] run the immutable Action or release wheel in both independent consumer repositories at the rc.1 pin; +- [ ] update status with exact `verified_source_sha` and workflow links; +- [ ] confirm P0 trust PRs 1–9 on the exact tag source and record attributable holdout aggregates; +- [ ] keep the package classifier at Beta until independent pilots and P0 closure meet the production gate. + +Promotion to **`v1.3.0`** additionally requires P0 closure + consumer + holdout evidence per [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). Do not re-attribute `v1.2.1` Sigstore evidence to typed-control-plane commits. + +## Related documents + +| Document | Purpose | +|---|---| +| [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | Authoritative R2 deep audit | +| [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md) | Sprint/PR execution program | +| [SOURCE_PROFILE_HARDENING.md](SOURCE_PROFILE_HARDENING.md) | Sprint 6 profile status | +| [HOLDOUT_LABEL_SEPARATION.md](HOLDOUT_LABEL_SEPARATION.md) | Sprint 8 prediction/eval split | +| [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) | Sprint 9 consumer pins | +| [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md) | Sprint 10 publication gate | +| [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) | Historical pre-control-plane audit | +| [STATUS.md](STATUS.md) | Command and lane inventory | +| [BACKENDS.md](BACKENDS.md) | Exact backend execution maturity and guarantee classes | +| [INTEGRATION.md](INTEGRATION.md) | Installation and GitHub Action setup | +| [RELEASE.md](RELEASE.md) | Maintainer release procedure | +| [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) | Independent advisory pilot process | +| [BENCHMARK.md](BENCHMARK.md) | Internal benchmark format and execution | diff --git a/docs/DEEP_AUDIT_2026-07-23_R2.md b/docs/DEEP_AUDIT_2026-07-23_R2.md index 80d1b89..d8a44f7 100644 --- a/docs/DEEP_AUDIT_2026-07-23_R2.md +++ b/docs/DEEP_AUDIT_2026-07-23_R2.md @@ -1,595 +1,189 @@ -# Open Verification Kernel Deep Audit — Revision 2 +# OVK Deep Audit R2 — 2026-07-23 -**Date:** 2026-07-23 -**Repository:** `fraware/open-verification-kernel` -**Scope:** current `main` after the latest engineer push and corrective changes made during this audit +Authoritative deep audit of Open Verification Kernel after the typed backend control-plane landing. This document supersedes day-to-day use of [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) for release judgment. The vision audit remains historical context. -## Executive judgment +Companion program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). Living dashboard: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). -The engineer push implemented the most important architectural transition in the OVK roadmap. The repository now contains a real typed backend control plane, backend-neutral obligations, typed routing decisions, registered adapters, backend-specific compilation, normalized execution attempts, fail-dominant aggregation, evidence v2, five enforceable lane paths, source-compiler profiles, template conformance generation, independent consumer repositories, holdout evaluation plumbing, and a materially stronger release-artifact chain. +## Final technical judgment -The complete vision is still not achieved. +The engineers completed the most important architectural transition in the roadmap. -The system is best described as an **advanced, policy-selectable verification kernel release candidate**. Backend selection controls execution inside the new control plane when a lane is explicitly enforced. The default product path remains shadow/legacy-authoritative. The current branch also contains several P0 trust defects that must be fixed before a new release can claim reliable enforced routing: +OVK now has a genuine typed backend control plane. In explicitly enforced lanes, a typed `RoutingDecision` identifies selected registered adapters, and those selected adapters compile and execute backend-specific obligations. The system includes backend-neutral obligations, typed routing, controlled attempts, conservative aggregation, evidence v2, five enforceable lane paths, source-compiler profiles, template conformance, consumer repositories, and holdout plumbing. -1. cache hits do not preserve the original execution attempt and can fabricate native-execution provenance from current tool availability; -2. execution-attempt identifiers include nondeterministic duration data; -3. a compiler/backend guarantee mismatch is silently rewritten instead of rejected; -4. backend selection ignores `coverage_requirements_met`; -5. fallback acceptance is represented as a broad boolean and does not enforce the configured fallback backend set; -6. self-protection metadata is trusted by default unless repository policy explicitly disables trust; -7. in-process adapters receive only soft timeout checks and cannot be cancelled; -8. evidence v2 and its schema do not yet make the complete routing and execution trace mandatory or cryptographically cross-bound; -9. the release and consumer evidence currently validate tag `v1.2.1`, whose commit predates the new control-plane implementation. +The vision is not fully achieved. Current code should be positioned as: -The release judgment is therefore: +**An advanced, policy-selectable verification-kernel release candidate for a future `v1.3.0-rc.1`.** -> **Current `main` is post-v1.2.1 development code and should be treated as a v1.3.0 release-candidate line. It is appropriate for internal testing and advisory pilots after current-source CI is green. It is not yet suitable for production-stable or broadly strict enforcement claims.** +The default product path remains shadow/legacy-authoritative. Enforced routing is available through lane policy, but several P0 trust properties remain incomplete. -## Audit basis and evidence provenance +## Audit coverage -The previous signed release is tag `v1.2.1` at commit: +This audit covered: -`a27d5720f4350c00bca34f71d991c31f5a2f38c7` - -GitHub Actions run `30010876652` successfully completed: - -- release verification; -- the package build; -- isolated wheel smoke; -- keyless Sigstore signing and verification. - -That run is valid evidence for the tagged `v1.2.1` source. - -It is not evidence for the current control-plane branch. The current branch is more than eighty commits ahead of that tag and adds the control plane, enforced adapters, source compilers, evidence v2, template conformance, holdout runner, consumer infrastructure, and later audit fixes. - -Two independent consumer repositories now exist: - -- `fraware/ovk-consumer-fastapi-terraform`; -- `fraware/ovk-consumer-express-actions`. - -They currently pin `fraware/open-verification-kernel@v1.2.1` and package version `1.2.1`. They are useful external-consumer scaffolds, but they do not validate the newly pushed control-plane source. - -The current branch must receive new attributable CI, native-backend, wheel, Action, consumer, and release-bundle evidence before publication. - -## Vision-achievement matrix - -| Capability | Current status | Judgment | -|---|---|---| -| Typed verification subject and materials | Implemented | Strong foundation | -| Backend-neutral obligation | Implemented for five lanes | Strong foundation | -| Typed backend registry | Implemented | Strong | -| Typed routing decision | Implemented | Strong data model | -| Selected backend controls execution | Implemented inside enforced control-plane path | Genuine architectural achievement | -| Default product path | Shadow/legacy-authoritative | Migration state, not universal enforcement | -| One authoritative route from inference to evidence | Not implemented | Legacy route and typed route coexist | -| Backend-specific compilation | Implemented for registered production adapters | Strong but guarantee validation incomplete | -| Fail-dominant aggregation | Implemented | Strong high-level semantics; fallback details incomplete | -| Backend execution cache | Routing and environment bound | Provenance loss on cache hits is P0 | -| Hard execution budgets | Partial | Subprocess worker exists; in-process adapters are soft bounded | -| Evidence v2 | Implemented | Model and schema require strengthening | -| Evidence quality | Broad invariant set implemented | Several invariants validate presence, not canonical recomputation | -| Release artifacts | Strong hash, attestation, provenance, and signing foundation | Cross-artifact material binding remains absent | -| Five enforced lanes | Available by policy | Default remains shadow | -| Authorization source compiler | Implemented as regex profile | Advisory-grade, not general strict-grade | -| Terraform plan compiler | Implemented as limited profile | Advisory/profile-limited | -| Kubernetes compiler | Implemented as limited profile | Advisory/profile-limited | -| GitHub Actions trust compiler | Implemented | Valuable advisory detector, incomplete semantics | -| Deployment compilers | Implemented for supplied abstractions | Bounded profile verification | -| Project-grounded CBMC | Scaffolding implemented | Full project compilation and harness traceability incomplete | -| Template catalog separation | Implemented | 95 catalog-only entries are honestly separated | -| Strict semantic conformance | Not implemented | Current gate primarily checks repository-link existence | -| Internal benchmark | Implemented | Regression signal, not independent accuracy | -| Holdout evaluation | Evaluator path implemented | Prediction generation and full isolation incomplete | -| Independent consumers | Repositories and pinned workflows exist | They validate the previous tag, not current source | -| Current-source release evidence | Not established | Required before next release | - -# 1. Kernel and routing audit - -## 1.1 What is now real - -`BackendRegistry` validates adapter and backend identities, rejects duplicates, validates capability manifests, and provides deterministic capability assessment. - -`BackendControlPlane.execute` iterates `RoutingDecision.selected`, retrieves only the named adapters, compiles backend-specific obligations, calculates environment fingerprints, executes adapters, normalizes results, records attempts, and aggregates the selected result set. - -This means selection can genuinely alter which backend runs. Authorization tests select either: - -- `authorization-deterministic`; -- `z3-native`. - -Self-protection can select: - -- `self-protection-deterministic`; -- `opa-native`. - -Infrastructure, CI secrets, and deployment each have a deterministic registered adapter. - -## 1.2 Dual routing remains - -`execute_kernel` still computes compatibility routing from inferred intents and static capability manifests before obligations are compiled. `adapter_runtime` later compiles a typed obligation and computes a second typed route for shadow or enforced execution. - -Consequences: - -- `KernelResult.routing` can differ from the route that produced enforced evidence; -- CLI and MCP planning surfaces can report a backend set different from the authoritative execution set; -- policy normalization and scoring are duplicated; -- routing identity is not a single immutable object from inference through attestation. - -Required resolution: - -1. infer candidate intents; -2. compile typed neutral obligations; -3. route each obligation exactly once through `route_obligation`; -4. pass those immutable decisions to execution; -5. return the actual routing decisions in `KernelResult`; -6. deprecate `route_intent` for internal execution. - -## 1.3 Coverage-blind selection - -Capability assessments include: - -- `material_requirements_met`; -- `coverage_requirements_met`. - -The typed router filters material requirements, guarantee compatibility, support, budget, and allow/deny policy. It does not reject a candidate whose `coverage_requirements_met` value is false. - -This allows a backend to become the required primary even when the adapter’s own capability assessment says the obligation lacks sufficient abstraction coverage. - -The evidence converter later downgrades an `allow` when overall obligation coverage is incomplete, which reduces immediate safety risk. It does not repair the routing inconsistency or prevent an unsupported backend execution from being represented as selected primary. - -Required behavior: - -- `coverage_requirements_met == false` must make the candidate ineligible for required-primary selection; -- partial support may be optional corroboration only when repository policy permits it; -- the rejected reason must state the exact coverage failure; -- quality validation must reject a required selection that contradicts capability assessment. - -## 1.4 Guarantee rewriting - -After an adapter compiles a backend obligation, the control plane compares the compiled `expected_guarantee` with the routing selection’s expected guarantee. When they differ, the current code rewrites the backend obligation to the router’s value. - -This is unsafe. The compiler is the component that knows which guarantee its generated payload can support. The router must select based on declared capability; it must not relabel a weaker or different compiled guarantee. - -Required behavior: - -- a mismatch produces an explicit `invalid_output` or `compiler_contract_error` attempt; -- no backend execution occurs for the mismatched obligation; -- the result requires human review; -- evidence-quality validation reports selected/compiled guarantee mismatch; -- tests include a malicious or defective adapter that returns a different guarantee. - -## 1.5 Fallback semantics are too broad - -Aggregation receives a single `fallback_accepted` boolean. When true, a required result whose guarantee is outside the obligation’s acceptable guarantee set can be accepted. - -The configured `FallbackPolicy.fallback_backends` and per-backend guarantee downgrade are not enforced by aggregation. - -Required behavior: - -- fallback acceptance must be evaluated per backend and guarantee; -- fallback is valid only when the selected backend is in the configured fallback set; -- the fallback guarantee must appear in an explicit accepted-fallback guarantee set; -- a native attempt that timed out or errored cannot be replaced by fallback in the same execution unless policy explicitly defines a second independent required attempt; -- evidence records the downgrade and why it was accepted. - -# 2. Execution identity and cache audit - -## 2.1 Attempt identifiers are nondeterministic - -`attempt_digest_input` excludes `started_at` and `finished_at`. It still includes `duration_ms`. - -Execution duration varies across equivalent runs. As a result: - -- equivalent execution attempts can receive different attempt IDs; -- evidence that embeds attempts can differ across runs; -- bundle and attestation digests can change even when inputs, tools, outputs, and decisions are identical. - -Required resolution: - -- remove `duration_ms` from the attempt identity; -- define attempt identity from backend obligation, backend, termination, native flag, tool identity, exit code, raw output digests, and worker image/tool digest; -- keep timing as observational metadata outside identity; -- add a test proving different timestamps and durations yield the same attempt ID; -- add a test proving changed termination, exit code, raw result, or tool digest changes the ID. - -## 2.2 Cache hits lose execution provenance - -The hardened cache stores `NormalizedBackendResult`, with metadata stored beside it. The control-plane cache interface returns only the normalized result. - -On a cache hit, the control plane creates a new `ExecutionAttempt` and sets `native_execution` from the current environment fingerprint’s `native_available` field. - -Tool availability is not proof that the cached result originally came from native execution. +- typed execution models and digest logic; +- backend registry, router, control plane, aggregation, and cache; +- all five enforced lanes; +- OPA, Z3, CBMC, Cedar, and external adapter surfaces; +- FastAPI, Express, Terraform, Kubernetes, GitHub Actions, deployment, and CBMC compilers; +- evidence v2, quality invariants, attestations, provenance, manifests, and Sigstore; +- GitHub Action, CI, release, holdout, and consumer workflows; +- FormalPR-Bench and template-conformance artifacts; +- both external consumer repositories. -Consequences: +## Release provenance correction -- deterministic cached results can be represented as native if the tool is now installed; -- native cached results can lose original tool version, exit code, timing, worker image, and termination provenance; -- quality checks can be satisfied by reconstructed current-state metadata instead of the original attempt. +The signed `v1.2.1` release is commit: -Required resolution: - -- cache a typed `CachedBackendExecution` containing the normalized result, original execution attempt, raw-result digest, environment fingerprint, adapter identity, and cache creation metadata; -- return that complete object on cache hit; -- preserve original `native_execution`, tool version, tool digest, exit code, termination, and attempt ID; -- add `cache_hit: true` as separate observational metadata; -- never synthesize original provenance from the current environment; -- invalidate all old backend-result cache entries by incrementing the cache schema version. - -## 2.3 In-process adapters are not hard bounded - -Several deterministic adapters run inside the OVK Python process. They compare elapsed time with the budget after the evaluator returns. - -A hung evaluator cannot be interrupted by that check. - -Required resolution: - -- run every authoritative adapter behind a worker boundary; -- use subprocess or spawned-process workers for deterministic adapters; -- enforce wall time outside adapter code; -- enforce memory and output limits; -- provide a minimal environment; -- prohibit repository writes unless policy permits them; -- record worker identity and termination reason. - -## 2.4 Worker environment defect fixed during this audit - -The worker described itself as an environment allowlist but inherited every parent variable except a finite denylist. Unknown credentials could therefore reach native tools. - -This audit changed the worker so: - -- only configured parent keys are inherited; -- explicit non-secret variables may be added by the caller; -- known credential keys remain forbidden; -- a non-positive timeout returns an immediate timeout without execution. - -New tests cover unknown parent credentials, explicit safe variables, and zero-budget execution. - -# 3. Self-protection trust audit - -The compiler supports trusted and untrusted before/after branch-protection materials. Enforced runtime currently defaults `metadata_trusted` to true unless repository policy explicitly sets it false. - -This is the wrong trust default. - -Metadata may originate from: - -- a PR-controlled file; -- current branch-protection collection that cannot reconstruct the removed base state; -- synthetic examples; -- an explicitly trusted base-branch collector. - -Required resolution: - -- default `metadata_trusted` to false; -- require a provenance object identifying collector, repository, revision, API endpoint or material source, collection time, and digest; -- only a protected base-branch workflow, signed external service, or explicit maintainer-supplied material may set trust true; -- current-state-only branch protection cannot authorize `allow` for gate preservation; -- untrusted complete-looking metadata must remain review-only; -- add adversarial tests where PR-head metadata falsely reports an unchanged gate. - -# 4. Evidence and artifact audit - -## 4.1 Evidence v2 is useful but not fully mandatory - -The Pydantic model adds typed control-plane fields, but many are optional. The JSON schema requires only a subset and allows broad additional properties. - -The schema does not require the complete trace: - -- compiler identity; -- materials; -- coverage; -- requested and eligible backends; -- attempted backends; -- execution attempts; -- routing-enforced state. - -Required resolution: - -- create evidence schema v2.1 or v3; -- require all control-plane fields for `routing_enforced: true` evidence; -- reference material, coverage, routing, and execution schemas instead of open objects; -- make selected backend entries typed so required and optional roles are preserved; -- make execution attempt linkage explicit through obligation and routing IDs. - -## 4.2 Invariants validate presence more often than canonical truth - -Examples: - -- routing ID is checked for presence but is not recomputed from embedded routing content; -- aggregate decision is not recomputed from typed selected roles and results; -- material digests are checked for shape/presence but not cross-verified against provenance and attestation; -- attempt-to-obligation linkage is weak because attempts do not carry obligation ID; -- selected backend names lose required/optional role information in evidence. - -Required resolution: - -- embed the typed routing decision or a canonical signed digest plus selected-role records; -- recompute routing ID during validation; -- recompute aggregate outcome from results and selected roles; -- add obligation ID and routing ID to attempts; -- verify material-set equality across obligation, evidence, provenance, and attestation. - -## 4.3 Cross-artifact material binding remains absent - -Define one canonical `material_set_digest` over sorted material references. - -Place it in: - -- `VerificationObligation`; -- `VerificationEvidence`; -- provenance predicate; -- attestation statement; -- attestation envelope or manifest metadata. - -Release verification must recompute and compare all values. - -## 4.4 CI-secrets material-size defect fixed during this audit - -The legacy CI-secrets compiler still used the digest string length as `size_bytes`. - -This audit migrated it to `material_reference_from_payload` and added a regression test binding size to canonical serialized bytes. - -# 5. Source compiler audit - -## 5.1 Authorization - -FastAPI and Express compilers create useful base/head IRs and source spans. They remain regex compilers. - -FastAPI risks include: - -- route decorator parsing stops at nested closing parentheses; -- dependency parsing can miss nested or multiline `Depends` expressions; -- included router and router prefix composition is incomplete; -- router identities are not module-qualified; -- duplicate route keys overwrite earlier entries; -- function analysis examines a bounded source suffix; -- absence of a detected unsupported pattern is treated as evidence of completeness. - -Express risks include: - -- middleware arguments are comma-split instead of AST parsed; -- nested calls and arrays are not modeled; -- import aliases and re-exports are incomplete; -- router prefix and mount resolution are file-local and name-based; -- duplicate route keys overwrite earlier entries; -- dynamic registrations can be missed. - -Judgment: - -> Both are source-referenced advisory profiles. They are not general strict-grade framework compilers. - -Required next implementation: - -- Python AST plus import/module graph for FastAPI; -- TypeScript compiler API or ESTree parser plus module graph for Express; -- route completeness accounting; -- explicit unresolved constructs; -- source-profile versioning; -- strict eligibility only for supported syntax subsets with independent corpus results. - -## 5.2 Infrastructure - -Terraform uses plan-shaped JSON, which is the correct strict-source boundary. Its present semantics are limited to top-level resource changes and a few generic exposure fields. - -Missing semantics include: - -- recursive child modules; -- `after_unknown` and sensitive values; -- provider defaults; -- resource dependencies; -- IAM policy effects; -- network routes and security groups; -- load balancer/listener relationships; -- provider-specific public exposure rules. +`a27d5720f4350c00bca34f71d991c31f5a2f38c7` -Kubernetes recognizes important object kinds, but currently: +Its release workflow successfully completed release verification, package build, isolated wheel smoke, and keyless Sigstore signing. -- every Ingress is treated as public; -- LoadBalancer and NodePort are treated as public without controller or address context; -- Gateway listener/class semantics are incomplete; -- NetworkPolicy and RBAC are stored but do not constrain reachability; -- selectors, namespaces, Services, Endpoints, Routes, and policy intersections are not fully resolved. +That evidence applies only to the old tag. The typed control plane and the bulk of the current architecture were added after that commit. The two external consumer repositories also currently pin `v1.2.1`, so they validate the old release, not the newly pushed implementation. -Judgment: +**Do not** re-attribute `v1.2.1` Sigstore/CI evidence to typed-control-plane commits. -> Useful advisory normalized profiles. Strict eligibility must be profile-specific and provider/controller-aware. +Latest `main` may be a benchmark badge commit marked `[skip ci]`. Generated benchmark data must not label a badge-only commit as `verified_source_sha` without a complete observed required-workflow set. -## 5.3 GitHub Actions +Terminology required going forward: -The compiler introduces the right trust-flow property, but currently treats every untrusted-trigger job or step as untrusted code. This is conservative and can create broad false positives. +| Field | Meaning | +|---|---| +| `benchmark_source_sha` | Source measured by FormalPR-Bench (or similar bench artifacts) | +| `verified_source_sha` | Source with a complete observed required-workflow set | -Missing semantics include: +Badge-only or `[skip ci]` commits must never be labeled verified. -- event-specific secret availability; -- job-level permissions overriding workflow permissions; -- reusable workflow input, permission, and secret propagation; -- `secrets: inherit` flow; -- environment protection configuration; -- semantic evaluation of `if:` conditions; -- checkout repository and ref trust; -- remote actions and workflows; -- expression data-flow and sanitization. +## What has genuinely been achieved -Judgment: +### Enforced backend execution -> Strong advisory trust detector, not a complete GitHub Actions authorization semantics engine. +Selected backends can now control execution within enforced lane paths. This is real, not merely metadata. -## 5.4 Deployment and CBMC +Current selectable pairs include: -Deployment compilers are bounded interpreters over supplied schemas and selected provider objects. They do not yet prove external controller state, artifact identity, reviewer decisions, rollback viability, or rollout metrics. +- authorization through `z3-native` or `authorization-deterministic`; +- self-protection through `opa-native` or `self-protection-deterministic`; +- infrastructure, CI secrets, and deployment through registered deterministic adapters. -CBMC has honest harness distinctions, but full project verification still requires source closure, compile database fidelity, changed-function mapping, generated environment models, unwind sufficiency, actual project compilation, and source-linked counterexamples. +### Strong execution and evidence foundations -# 6. Template conformance audit +The repository now contains: -The conformance matrix correctly separates 95 catalog-only templates from five linked production lanes. +- typed verification subjects and materials; +- backend-neutral obligations; +- backend capability assessments; +- typed routing decisions; +- backend-specific obligations; +- raw executions and normalized results; +- execution budgets; +- evidence v2; +- fail-dominant aggregation; +- routing- and environment-bound cache keys; +- release manifests, provenance, attestations, HMAC, and Sigstore support. -Its `strict_eligible` status is still too strong. The gate checks existence of: +### Honest catalog separation -- intent file; -- evaluator; -- compiler; -- registry; -- pass example; -- fail example; -- enforcement test. +The template-conformance system distinguishes five linked production lanes from 95 catalog-only templates. This is substantially more honest than presenting all 100 templates as operational. Remaining gap: `strict_eligible` is still inferred mainly from repository links, fixtures, and a test file, rather than semantic completeness. -It does not establish: +### External consumer infrastructure exists -- malformed and unknown behavior; -- compiler coverage completeness; -- source-material acquisition; -- selected/executed backend consistency; -- native versus deterministic guarantee strength; -- counterexample correctness; -- artifact integrity; -- external calibration. +The FastAPI/Terraform and Express/GitHub Actions consumer repositories are real and include pinned Action workflows, scenario matrices, pilot ledgers, release-bundle paths, and wheel-install scripts. Their automated scenarios remain separate from the required human-adjudicated pilot gate. -Required status model: +## P0 defects remaining -- `catalog_only`; -- `executable_advisory`; -- `source_profile_strict_eligible`; -- `externally_calibrated_strict`; -- `deprecated`. +### 1. Attempt IDs are nondeterministic -No template should be `source_profile_strict_eligible` unless its source compiler’s supported subset, coverage requirements, negative corpus, unknown corpus, and artifact checks are generated and tested. +Execution attempt identity still includes `duration_ms`. Timing varies between otherwise equivalent runs, so attempt IDs, evidence, bundle IDs, and attestations can vary without a semantic input or output change. -# 7. Holdout and external validation audit +**Required fix:** remove timing fields from canonical attempt identity; prove stable IDs across sequential, parallel, cached, and uncached equivalent executions. -## 7.1 Holdout runner defects fixed during this audit +### 2. Cache hits lose original execution provenance -The downloaded holdout artifact contained an executable `harness/evaluate.py`. +The cache returns normalized results without the complete original attempt. On a cache hit, the control plane synthesizes a new attempt and may re-infer `native_execution` from current tool availability. -Before this audit: +This can falsely describe a cached deterministic result as native after a tool is installed, or erase native tool provenance after the tool disappears. -- remote assets were not checked against an independent digest; -- Python 3.10/3.11 fell back to unrestricted `tar.extractall`; -- archive links and special files were not rejected; -- the downloaded evaluator inherited `HOLDOUT_DOWNLOAD_TOKEN` and runner environment; -- aggregate output was not fully validated against the JSON schema; -- `leakage_guard.fail_closed` was not required to be true. +**Required fix:** store and replay `CachedBackendExecution` with original attempt, native flag, tool version/digest, termination/exit code, raw-result digest, environment fingerprint, and normalized result. Bump cache schema to v3. -This audit changed the runner and workflow to require: +### 3. Guarantee mismatches are silently rewritten -- an independently supplied SHA-256 for remote assets; -- path-safe manual extraction; -- rejection of links, devices, FIFOs, and special members; -- isolated Python execution with a minimal environment and no tokens; -- full Draft 2020-12 schema validation; -- strict leakage guard enforcement; -- new supply-chain security tests. +When the router expects one guarantee and the adapter compiler produces another, the control plane rewrites the compiled obligation to the router’s expectation. -Remaining holdout work: +**Required fix:** treat mismatch as a compiler-contract error; skip execution; require review. Never silent `model_copy` rewrite. -- separate prediction and evaluation jobs; -- generate predictions from the exact current wheel/commit without labels; -- digest or sign predictions; -- record wheel, source, holdout asset, prediction, and aggregate digests; -- use a sandbox/container with network disabled for evaluator execution; -- publish only aggregate metrics. +### 4. Coverage does not govern backend eligibility -## 7.2 Independent consumers exist but validate the old release +Adapters report `coverage_requirements_met`, but required-primary selection does not reject candidates for which this field is false. -The two consumer repositories are real and contain: +**Required fix:** reject incomplete candidates as required primaries; allow only as optional corroborators under explicit policy. Stop hardcoding `coverage_requirements_met=True` in manifest assessment. -- immutable Action pins; -- scenario matrices; -- advisory and strict workflow definitions; -- pilot ledgers; -- wheel-install scripts; -- fork simulations. +### 5. Fallback policy is under-specified -They currently pin `v1.2.1` at commit `a27d572...`. +Aggregation receives a broad `fallback_accepted` boolean. It does not constrain fallback by backend, guarantee type, or failure cause. -The control-plane implementation is post-tag development. Therefore: +**Required fix:** backend-, guarantee-, and cause-specific policy. Native timeout, tool error, invalid output, and resource exhaustion must never become passing fallback results. -- consumer existence is established; -- current control-plane validation is not established; -- automated scenario entries are not human-adjudicated pilots; -- cross-fork and comment/check-run behavior still require attributable workflow evidence. +### 6. Self-protection metadata defaults to trusted -Required next step: +Enforced self-protection currently initializes `metadata_trusted` to true unless policy explicitly sets it false. -1. publish or pin an immutable current-source release candidate; -2. update both consumers; -3. dispatch every scenario workflow; -4. retain artifacts and run conclusions; -5. ingest results into the pilot ledger; -6. complete human adjudication targets before production claims. +**Required fix:** default `metadata_trusted=False`. Trust only from protected base-workflow provenance, signed service, or explicit maintainer-supplied material. Current-state-only branch protection cannot set trusted. -# 8. Release and CI assessment +### 7. Deterministic adapters cannot be hard-cancelled -The green `v1.2.1` release run is valid for the old tag only. +Several deterministic adapters execute in-process and inspect elapsed time only after returning. A hung or unexpectedly expensive evaluator cannot be terminated. -Current `main` must pass: +**Required fix:** every authoritative adapter must run behind a subprocess/spawned-worker boundary with wall-time, output, env, path, and hard cancellation. -- full unit and integration tests; -- lint; -- all release-preflight checks; -- template conformance; -- wheel-outside-checkout smoke; -- Action automatic-diff dogfood; -- strict-block Action dogfood; -- native OPA, Z3, and CBMC jobs; -- Cedar honesty probe; -- release-bundle adversarial checks; -- holdout security tests; -- consumer release-candidate scenarios. +### 8. Two routing paths remain -Badge artifacts currently call the benchmark source SHA `verified_source_sha`. The badge workflow can produce that field after a benchmark-only workflow even when general CI is absent. Rename or split this provenance: +The kernel calculates compatibility routing before compiling obligations. The enforced runtime later compiles a typed obligation and calculates another typed route. -- `benchmark_source_sha` means the source evaluated by FormalPR-Bench; -- `verified_source_sha` is present only when an exact successful required workflow set is recorded; -- include workflow run IDs and conclusions in release status. +**Required fix:** compile typed obligations before routing; route each obligation exactly once; execute that immutable `RoutingDecision`; expose the same `routing_id` through kernel, CLI, MCP, evidence, provenance, and attestation. -# 9. Direct changes made during this audit +### 9. Evidence v2 does not require the complete trace -1. Fixed CI-secrets material byte-size integrity. -2. Added CI-secrets material-size regression coverage. -3. Changed backend subprocess workers from denylist inheritance to a minimal parent-environment allowlist. -4. Added worker tests for unknown credential isolation, explicit safe variables, and non-positive timeout behavior. -5. Required immutable SHA-256 verification for remote holdout assets. -6. Replaced unsafe tar extraction with path-safe extraction and special-file rejection. -7. Removed tokens and inherited credentials from downloaded holdout evaluator execution. -8. Added full holdout aggregate schema validation and stricter leakage guards. -9. Updated the holdout workflow to install runner dependencies, require the asset digest, and validate the sanitized aggregate. -10. Added holdout path traversal, symlink, token isolation, digest, and leakage tests. +The schema requires obligation ID, routing ID, selected and executed backends, and aggregation policy. It leaves compiler, materials, coverage, requested/eligible/attempted backends, execution attempts, and routing-enforced state optional or broadly typed. -All changes require a fresh CI run on the current source. +**Required fix:** evidence v3 with mandatory full control-plane trace and one canonical material-set digest cross-bound across obligation, evidence, provenance, attestation, and release verification. -# 10. Updated release verdict +## Direct fixes required with this audit baseline -## Suitable after current-source CI is green - -- local development; -- internal evaluation; -- shadow-mode deployment; -- advisory pilots for the five bounded lanes; -- evidence and artifact integrations; -- controlled lane-specific enforced experiments. +Claimed during audit authoring; must be present in tree before Sprint 0 measurement: -## Suitable only after repository-specific calibration - -- lane-specific strict required checks where: - - material acquisition is trusted; - - the compiler profile is explicitly supported; - - coverage is sufficient; - - the enforced backend has calibrated behavior; - - fallback is disabled or explicitly bounded. +1. **CI-secrets / material integrity** — `size_bytes` must bind canonical serialized payload length, not digest string length. +2. **Backend worker environment** — inherit only configured safe parent vars; strip credentials; reject non-positive wall budget. +3. **FormalPR-Holdout supply-chain boundary** — immutable SHA-256, path-safe extraction, isolated Python, no tokens in evaluator, schema validation, leakage guards. -## Not yet supportable as a general claim +## Definition of completed vision (18-condition gate) -- production-stable enforcement for arbitrary repositories; -- complete formal verification of arbitrary pull requests; -- strict-grade FastAPI or Express coverage outside a documented supported subset; -- provider-complete Terraform or Kubernetes reachability; -- complete GitHub Actions trust semantics; -- native execution across all ten advertised tools; -- external accuracy derived from FormalPR-Bench; -- current-control-plane validation from the `v1.2.1` tag or its consumers. +Ship / promote to `v1.3.0` only when all hold: -# 11. Bottom line +1. one route per enforced obligation; +2. the same `routing_id` across kernel, evidence, provenance, and attestation; +3. coverage-aware selection; +4. no silent guarantee mismatch; +5. constrained fallback; +6. provenance-preserving cache hits; +7. deterministic attempt identities; +8. hard-bounded adapters; +9. protected self-protection trust; +10. complete evidence trace; +11. cross-artifact material binding; +12. semantic template conformance; +13. current-source CI on a non-`[skip ci]` SHA; +14. current wheel and Action validation in both consumers; +15. label-separated holdout predictions; +16. immutable retained evaluation artifacts; +17. attributable release gates on the exact tag source; +18. correct `benchmark_source_sha` vs `verified_source_sha` terminology in published artifacts. -The engineers achieved the architectural core that was previously missing: selected registered backends can now control execution inside the enforced control plane. The remaining work is no longer primarily about adding architectural nouns. It is about making the control plane trustworthy under cache reuse, compiler disagreement, incomplete coverage, fallback, untrusted metadata, execution budgets, cross-artifact verification, source semantics, and release provenance. +## Related documents -The next release must not reuse the `v1.2.1` evidence story. The new architecture requires a new attributable release-candidate cycle, updated consumer pins, current-source CI, native backend evidence, signed release artifacts, and end-to-end holdout and pilot results. +| Document | Role | +|---|---| +| [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md) | Sprint/PR execution program | +| [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) | Living adoption dashboard | +| [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) | Historical pre-control-plane audit | +| [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) | Consumer pin checklist | +| [FORMALPR_HOLDOUT_GOVERNANCE.md](FORMALPR_HOLDOUT_GOVERNANCE.md) | Holdout governance | diff --git a/docs/ENGINEERING_PROGRAM_2026-07-23_R2.md b/docs/ENGINEERING_PROGRAM_2026-07-23_R2.md index fad827d..5fbf887 100644 --- a/docs/ENGINEERING_PROGRAM_2026-07-23_R2.md +++ b/docs/ENGINEERING_PROGRAM_2026-07-23_R2.md @@ -1,1284 +1,181 @@ -# Open Verification Kernel Engineering Program — Revision 2 +# OVK Engineering Program R2 — 2026-07-23 -**Purpose:** standalone implementation instructions for completing the OVK vision after the latest control-plane push -**Repository:** `fraware/open-verification-kernel` -**Companion audit:** `docs/DEEP_AUDIT_2026-07-23_R2.md` +Execution program derived from [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Release positioning: future **`v1.3.0-rc.1`** candidate. Do not re-validate signed `v1.2.1` (`a27d572…`) as if it included the typed control plane. -# 1. Program mandate +## Mandatory constraints -The next engineering phase must convert the current post-v1.2.1 control-plane branch into a release-substantiated, internally consistent, externally validated verification kernel. +- Do not merge trust-chain PRs out of order (identity → cache → contracts → single route → isolation → evidence). +- Every PR must include the eight analysis sections below and fail-closed adversarial tests. +- Sprints 6–7 may overlap late PRs only after PR6 lands. +- External consumer and holdout work needing write access outside this repo is coordinated separately; this repo carries pins, checklists, and scaffolding. -The architecture already contains the right major components: +## Per-PR analysis checklist (required) -- backend-neutral obligations; -- typed capability assessments; -- typed routing decisions; -- backend-specific compilation; -- registered adapters; -- controlled execution records; -- normalized evidence; -- conservative aggregation; -- evidence-quality validation; -- release manifests, attestations, provenance, and signatures. +Every PR in this program must document: -The remaining work must establish that these components preserve trust under: +1. **Architecture** — what trust boundary or control-plane invariant changes. +2. **Schema** — which schemas/versions bump and why. +3. **Cache** — key/value identity impact; invalidation or migration. +4. **Migration** — how old artifacts/entries fail closed or upgrade. +5. **Trust-boundary** — what an adversary can no longer forge or confuse. +6. **Adversarial tests** — explicit negative tests that fail closed. +7. **Artifacts** — which evidence/provenance/attestation/release fields change. +8. **Docs** — status, roadmap, and operator-facing honesty updates. -- cache reuse; -- nondeterministic timing; -- incomplete abstraction coverage; -- compiler/backend disagreement; -- backend failure and fallback; -- untrusted metadata; -- resource exhaustion; -- malicious artifacts; -- source-compiler incompleteness; -- independent consumer use; -- release publication. +## Sprint 0 — Current-source baseline -Do not add more adapter names or catalog templates until the P0 control-plane and release invariants below are complete. +On one **non-`[skip ci]`** source SHA, run and retain: -# 2. Product and release boundary +- general CI, native Tier 1, package/wheel smoke outside checkout; +- Action dogfood, release preflight, expanded FormalPR-Bench; +- template conformance, adversarial release-bundle checks. -## 2.1 Version boundary +Record exact workflow run URLs in [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). -Treat current `main` as the development line for: +Introduce `benchmark_source_sha` alongside `verified_source_sha` in badge/summary renderers and schemas. `verified_source_sha` requires a complete observed required-workflow set; badge-only commits must not be labeled verified. -`v1.3.0-rc.1` +## Sprint 1 — Attempt identity and cache provenance -Do not describe the current branch as the already released `v1.2.1` implementation. +### PR1 — Deterministic attempt identity -`v1.2.1` at commit `a27d5720f4350c00bca34f71d991c31f5a2f38c7` remains the previous signed release. The typed control-plane implementation was added afterward. +**Targets:** `ovk/core/execution_models.py` (`attempt_digest_input` / `compute_attempt_id`) -## 2.2 Supported production profiles +**Acceptance:** -The next release candidate may advertise five bounded profiles: +- Remove `duration_ms` (and any other timing) from canonical attempt identity; keep duration as observational metadata only. +- Prove stable IDs across sequential, parallel, cached, and uncached equivalent runs. +- Remove/replace vacuous tests that assert timing participates in identity. -- self-protection; -- authorization; -- infrastructure exposure; -- CI secret exposure; -- deployment approval state. +### PR2 — Provenance-preserving cache v3 -The release candidate must distinguish: +**Targets:** `ovk/core/result_cache.py`, `ovk/core/backend_control_plane.py` cache-hit branch -- legacy-authoritative execution; -- shadow control-plane execution; -- lane-specific enforced execution; -- source-profile strict eligibility; -- externally calibrated strict eligibility. +**Acceptance:** -## 2.3 Claims prohibited until later gates +- Introduce `CachedBackendExecution` storing original `ExecutionAttempt`, `native_execution`, tool version/digest, termination/exit code, raw-result digest, environment fingerprint, normalized result. +- On hit: **replay** stored attempt (do not synthesize; do not re-infer `native_execution` from current tool availability). +- Bump schema to `ovk.cache.v3`; migrate or invalidate v2 entries explicitly. +- Adversarial tests: install/remove native tool between write and hit; prove provenance unchanged. -Do not claim: +## Sprint 2 — Routing-contract enforcement -- production-stable enforcement across arbitrary repositories; -- complete formal verification of arbitrary changes; -- strict-grade support for all FastAPI, Express, Terraform, Kubernetes, GitHub Actions, or deployment semantics; -- native execution for Cedar, TLA+, Kani, Dafny, Verus, Lean, or Alloy; -- external accuracy from FormalPR-Bench; -- current-control-plane validation from the `v1.2.1` release or consumer pins. +### PR3 — Coverage and guarantee enforcement -# 3. Program invariants +**Targets:** `ovk/core/router.py`, `ovk/core/backend_control_plane.py` -Every pull request must preserve these invariants. +**Acceptance:** -## 3.1 Selection integrity +- Reject candidates with `coverage_requirements_met=False` as required primaries; allow only as optional corroborators under explicit policy. +- Stop hardcoding `coverage_requirements_met=True` in `_manifest_assessment`. +- On compiler vs router `expected_guarantee` mismatch: compiler-contract error, skip execution, require review — never silent rewrite. -A backend can affect a decision only when it was: +### PR4 — Fallback policy v2 -1. registered; -2. capability-assessed; -3. eligible; -4. selected; -5. compiled successfully; -6. attempted; -7. normalized successfully; -8. accepted by evidence-quality validation. +**Targets:** `ovk/core/execution_models.py` (`FallbackPolicy`), `ovk/core/backend_aggregation.py`, evidence decision fields for INV-017 -An unselected backend must never affect the decision. +**Acceptance:** -## 3.2 Coverage integrity +- Replace broad `fallback_accepted=allow_fallback` with backend-, guarantee-, and cause-specific policy. +- Native timeout, tool error, invalid output, and resource exhaustion must never become passing fallback results. -A backend whose assessment has `coverage_requirements_met == false` cannot be a required primary backend. +### PR5 — Self-protection trust provenance -A source compiler cannot produce strict `allow` unless: +**Targets:** `ovk/core/adapter_runtime.py`, `ovk/core/self_protection_compiler.py` -- its supported source profile is identified; -- required source materials are complete; -- unsupported constructs are absent or explicitly accepted by policy; -- coverage status is complete; -- the minimum profile confidence threshold is satisfied. +**Acceptance:** -## 3.3 Guarantee integrity +- Default `metadata_trusted=False`. +- Trust only from protected base-workflow provenance, signed service, or explicit maintainer-supplied material. +- Current-state-only branch protection cannot set trusted. -The adapter compiler determines the guarantee supported by its backend obligation. +## Sprint 3 — One authoritative route -The router cannot relabel that guarantee. +### PR6 — Single authoritative routing pipeline -When routing and compilation disagree: +**Targets:** `ovk/core/router.py`, kernel/CLI/MCP planning surfaces, evidence/provenance/attestation emitters -- do not execute the backend; -- emit `compiler_contract_error` or `invalid_output`; -- require human review; -- preserve the disagreement as a quality issue. +**Acceptance:** -## 3.4 Cache provenance integrity +- Compile typed obligations **before** routing; route each obligation exactly once; execute that immutable `RoutingDecision`. +- Eliminate dual outcomes from `route_intent` (compat) vs `route_obligation` (typed). +- Same `routing_id` across kernel, CLI, MCP, evidence, provenance, attestation. -A cache hit must preserve the original: +## Sprint 4 — Hard adapter isolation -- execution attempt; -- native-execution state; -- tool version and digest; -- termination; -- exit code; -- raw-result digest; -- worker identity; -- normalized result. +### PR7 — Isolated deterministic workers -Current tool availability cannot be used to reconstruct prior execution provenance. +**Targets:** five `*DeterministicAdapter.run` classes; `ovk/core/execution_budget.py` -## 3.5 Fallback integrity +**Acceptance:** -Fallback is valid only when: +- Move deterministic evaluators behind subprocess/spawned-worker boundary with wall-time, output, env, path, and hard cancellation. +- Control plane always passes worker; in-process-only authoritative adapters are forbidden. -- the backend is explicitly listed in `fallback_backends`; -- the fallback guarantee is explicitly accepted; -- the required backend was unavailable before execution or policy explicitly permits an independent fallback attempt; -- a native timeout, crash, invalid output, or resource exhaustion is not silently replaced by a passing fallback result. +### PR8 — Remaining adapter isolation -## 3.6 Material integrity +**Acceptance:** -One canonical material set must bind: +- Native OPA/Z3/CBMC/Cedar paths share the same externally enforced worker contract. +- No authoritative adapter may inspect elapsed time only after returning. -- obligation; -- evidence; -- provenance; -- attestation; -- release verification. +## Sprint 5 — Evidence v3 and material binding -## 3.7 Release provenance integrity +### PR9 — Evidence v3 and material-set binding -A `verified_source_sha` exists only when the exact SHA has an observed successful required workflow set. +**Targets:** evidence schema v3, `ovk/core/evidence_invariants.py`, provenance/attestation/release verification -A benchmark-only run may record `benchmark_source_sha`. It must not create release-verification provenance. +**Acceptance:** -# 4. Workstream ownership +- Require full control-plane trace: compiler, materials, coverage, requested/eligible/attempted backends, execution attempts, routing-enforced state. +- Cross-bind one canonical **material-set digest** across obligation, evidence, provenance, attestation, and release verification; recompute identities during validation. -Assign clear owners for four workstreams. +## Sprint 6 — Source-profile hardening -## Workstream A — Kernel correctness +AST/module-graph authorization profiles; recursive Terraform plans; controller-aware K8s reachability; deeper Actions permissions/secret flow; deployment strictness only on explicit trusted profiles. Touch lane compilers under `ovk/core/*_compiler.py` and source-profile modules. -Owns: +Parallelizable after PR6. -- execution models; -- router; -- registry; -- control plane; -- cache; -- aggregation; -- worker isolation; -- migration runtime. +## Sprint 7 / PR10 — Semantic template conformance v2 -## Workstream B — Evidence and supply chain +Replace file-existence `strict_eligible` generation with: -Owns: +`catalog_only` | `executable_advisory` | `source_profile_strict_eligible` | `externally_calibrated_strict` | `deprecated` -- evidence schemas; -- invariants; -- rendering; -- provenance; -- attestation; -- release bundle; -- Sigstore; -- release workflows; -- verified source records. +Every status must derive from executed semantic evidence, not repo-link/fixture/test-file presence. -## Workstream C — Semantic compilers +## Sprint 8 — Label-separated holdout -Owns: +Generate predictions from the exact RC artifact **without** labels; sign/digest; evaluate separately with protected labels; publish aggregates only. Builds on Phase A holdout hardening. -- authorization profiles; -- Terraform and Kubernetes profiles; -- GitHub Actions trust semantics; -- deployment profiles; -- project-grounded CBMC; -- compiler coverage and corpus governance. +## Sprint 9 — Consumer validation on current code -## Workstream D — External evaluation +Update both consumers from `v1.2.1` to immutable `v1.3.0-rc.1` (or audited commit): -Owns: +- https://github.com/fraware/ovk-consumer-fastapi-terraform +- https://github.com/fraware/ovk-consumer-express-actions -- template conformance; -- FormalPR-Bench governance; -- FormalPR-Holdout; -- consumer repositories; -- pilot ledgers; -- adjudication; -- release-candidate validation. +Dispatch workflows, download evidence, verify bundles, exercise true cross-fork PRs; keep human pilot ledgers separate from automated fixtures. Update [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) pins in this repo; consumer repo PRs require write access. -The execution and evidence contracts remain centrally reviewed. Compiler teams must not create local replacements for routing, attempt, result, or material models. +## Sprint 10 — Attributable publication -# 5. Sprint 0 — Establish the current-source baseline +- Correct benchmark vs verified-source terminology in published artifacts. +- Require all automated release gates on the exact tag source. +- Sign artifacts; retain workflow IDs and digests. +- Promote to `v1.3.0` only after P0 closure (PRs 1–9), consumer validation, and attributable holdout aggregates. -## Objective +## Required PR order (summary) -Produce attributable evidence for the current post-audit source before making additional architectural changes. +1. Deterministic attempt identity +2. Provenance-preserving cache v3 +3. Coverage and guarantee enforcement +4. Fallback policy v2 +5. Self-protection trust provenance +6. Single authoritative routing pipeline +7. Isolated deterministic workers +8. Remaining adapter isolation +9. Evidence v3 and material-set binding +10. Template conformance v2 -## Required actions +## Phase A prerequisite (before Sprint 0 measurement) -1. Select one non-badge source commit containing: - - the latest engineer push; - - CI-secrets material-size correction; - - minimal worker environment correction; - - holdout supply-chain corrections; - - the Revision 2 audit and engineering program. -2. Disable or avoid benchmark badge commits while baseline workflows run. -3. Run: - - general CI; - - native Tier 1; - - package/wheel smoke; - - automatic-diff Action dogfood; - - strict-block Action dogfood; - - release preflight; - - expanded FormalPR-Bench; - - template conformance; - - release-bundle adversarial tests. -4. Retain: - - JUnit or test output; - - benchmark summary; - - wheel artifact; - - Action evidence artifacts; - - native backend artifacts; - - release-preflight report. -5. Record exact source SHA and run IDs in `docs/CURRENT_RELEASE_STATUS.md`. - -## Required corrections before the baseline can be called green - -- update any test that assumes inherited worker environment; -- ensure `jsonschema` is installed for holdout runner tests; -- supply a holdout asset digest only in the optional holdout workflow; -- verify that Python isolated mode can execute the frozen holdout evaluator; -- resolve any package-data regression introduced by the current source. - -## Acceptance criteria - -- all required jobs are green on the same source SHA; -- no health statement cites a `[skip ci]` badge commit; -- generated benchmark files contain `benchmark_source_sha`; -- `verified_source_sha` is omitted until required workflows are verified. - -# 6. Sprint 1 — Correct execution identity and cache provenance - -## Objective - -Make execution identity deterministic and cache reuse provenance-preserving. - -## 6.1 Remove observational timing from attempt identity - -### Files - -- `ovk/core/execution_models.py` -- `tests/test_execution_models.py` -- `tests/test_adversarial_control_plane.py` - -### Required changes - -Change `attempt_digest_input` so it excludes: - -- `attempt_id`; -- `started_at`; -- `finished_at`; -- `duration_ms`. - -Retain identity-bearing fields: - -- backend obligation ID; -- backend; -- required role; -- termination; -- native execution; -- tool version; -- tool digest; -- worker image digest; -- exit code; -- stdout digest; -- stderr digest; -- raw result digest. - -### Tests - -- different start, finish, and duration values produce identical attempt IDs; -- different termination changes the ID; -- different raw result changes the ID; -- different tool digest changes the ID; -- sequential and parallel equivalent executions produce the same bundle ID. - -## 6.2 Cache the complete execution provenance - -### Files - -- `ovk/core/result_cache.py` -- `ovk/core/backend_control_plane.py` -- `ovk/core/execution_models.py` -- `tests/test_cache_worker_control_plane.py` -- `tests/test_verification_cache.py` - -### New model - -Add: - -```python -class CachedBackendExecution(BaseModel): - schema_version: Literal["ovk.cached_backend_execution.v1"] - execution_attempt: ExecutionAttempt - normalized_result: NormalizedBackendResult - environment_fingerprint: BackendEnvironmentFingerprint - raw_result_digest: str | None - cached_at_unix_ms: int -``` - -### Required behavior - -- store the complete object; -- return the complete object; -- reuse the original attempt ID and native flag; -- add `cache_hit: true` as separate metadata without altering original execution provenance; -- do not create a new completed native attempt on cache hit; -- increment `CACHE_SCHEMA_VERSION`; -- delete or ignore all previous backend-result cache entries. - -### Required adversarial tests - -1. Execute deterministic backend with native tool unavailable, cache it, install/mock tool availability, read cache, confirm `native_execution` remains false. -2. Execute native backend, cache it, remove/mock tool unavailable, read cache, confirm original native provenance and tool version remain intact. -3. Change tool digest and confirm cache miss. -4. Corrupt cached attempt and confirm cache miss or deletion. -5. Confirm cache hit does not change evidence or bundle identity. - -## 6.3 Remove vacuous cache tests - -Replace any assertion of the form: - -```python -assert condition or True -``` - -with a direct identity assertion. - -The policy-digest test must prove different policy digests produce different obligation IDs, routing IDs, or cache key components. - -## Sprint 1 exit criteria - -- attempt identity is deterministic; -- cache reuse preserves original execution provenance; -- no test can pass unconditionally; -- evidence and bundle identities are stable across cache misses and hits. - -# 7. Sprint 2 — Enforce routing, coverage, guarantee, and fallback contracts - -## Objective - -Make the typed router and control plane reject inconsistent execution plans instead of repairing metadata silently. - -## 7.1 Coverage-aware eligibility - -### Files - -- `ovk/core/router.py` -- `ovk/core/backend_registry.py` -- all production adapters’ `can_handle` implementations -- `tests/test_typed_router.py` -- `tests/test_authorization_enforcement.py` -- `tests/test_remaining_lane_enforcement.py` - -### Required behavior - -A candidate is eligible as required primary only when: - -- `support == supported`; -- material requirements are met; -- coverage requirements are met; -- guarantee is accepted; -- budget permits execution; -- backend is allowed; -- backend is available or an accepted pre-execution fallback path exists. - -When coverage is insufficient: - -- candidate moves to rejected; -- rejection reason includes coverage status and warnings; -- optional corroboration is permitted only by explicit policy. - -### Tests - -- candidate with false coverage cannot be selected required; -- complete candidate wins over higher-scored incomplete candidate; -- all incomplete candidates produce no required selection and review; -- partial optional corroborator cannot upgrade a required unknown; -- source compiler with unsupported constructs cannot produce strict allow. - -## 7.2 Reject guarantee mismatch - -### Files - -- `ovk/core/backend_control_plane.py` -- `ovk/core/evidence_invariants.py` -- `tests/test_adversarial_control_plane.py` - -### Required behavior - -Replace guarantee rewriting with explicit failure. - -When: - -```text -compiled.expected_guarantee != selected.expected_guarantee -``` - -create an execution record with: - -- termination `invalid_output` or new `compiler_contract_error`; -- status `error`; -- human review; -- no backend execution; -- quality issue `OVK-INV-GUARANTEE-MISMATCH`. - -### Tests - -- defective adapter compiles weaker guarantee; -- defective adapter compiles unrelated guarantee; -- router selection remains unchanged but execution is rejected; -- no rewritten guarantee appears in evidence. - -## 7.3 Make fallback per-backend and per-guarantee - -### Files - -- `ovk/core/execution_models.py` -- `ovk/core/backend_aggregation.py` -- `ovk/core/router.py` -- `schemas/backend.routing.schema.json` -- `tests/test_authorization_enforcement.py` -- `tests/test_self_protection_enforcement.py` -- new `tests/test_fallback_policy.py` - -### Model change - -Extend fallback policy: - -```python -class AcceptedFallback(BaseModel): - backend: str - guarantee_type: str - permitted_termination_causes: list[Literal["tool_unavailable"]] - -class FallbackPolicy(BaseModel): - allow_fallback: bool - accepted: list[AcceptedFallback] -``` - -Do not accept timeout, tool error, invalid output, or resource exhaustion as fallback causes in the initial policy. - -### Tests - -- listed fallback backend and guarantee can satisfy a pre-execution unavailable primary; -- unlisted fallback backend cannot; -- listed backend with wrong guarantee cannot; -- timeout never becomes fallback pass; -- tool error never becomes fallback pass; -- evidence records fallback cause and guarantee downgrade. - -## 7.4 Default self-protection metadata to untrusted - -### Files - -- `ovk/core/self_protection_compiler.py` -- `ovk/core/adapter_runtime.py` -- `ovk/core/context.py` -- `schemas/verification.config.schema.json` -- `tests/test_self_protection_enforcement.py` -- `tests/test_trusted_policy_loading.py` - -### Required behavior - -- compiler default `metadata_trusted=False`; -- runtime default false; -- trust true requires typed provenance from protected base workflow or signed service; -- current-state-only metadata cannot authorize allow; -- explicit examples and fixtures may opt in to trusted metadata only inside tests. - -### Required provenance - -```python -class MetadataProvenance(BaseModel): - collector: str - source: Literal["protected_base_workflow", "signed_service", "maintainer_input"] - repo: str - base_sha: str - head_sha: str - collected_at: str - sha256: str -``` - -## Sprint 2 exit criteria - -- coverage affects selection; -- guarantee mismatch cannot be hidden; -- fallback is narrowly enforced; -- self-protection trust is fail-closed by default. - -# 8. Sprint 3 — Unify routing from kernel inference through evidence - -## Objective - -Remove the dual-route architecture. - -## Files - -- `ovk/core/kernel.py` -- `ovk/core/check.py` -- `ovk/core/obligation_compiler.py` -- `ovk/core/router.py` -- `ovk/core/adapter_runtime.py` -- MCP planning and execution surfaces -- CLI plan/check/run surfaces - -## Required architecture - -1. Build repository context. -2. Infer candidate intents. -3. Compile typed neutral obligations. -4. Route each obligation exactly once. -5. Execute those exact routing decisions. -6. Return the exact typed decisions in `KernelResult`. -7. Bind routing IDs into evidence and artifacts. - -`route_intent` may remain only as: - -- deprecated compatibility API; -- catalog exploration API; -- never an internal execution route. - -## Kernel result model - -Replace compatibility routing dictionaries with: - -```python -class KernelResult(BaseModel): - plan: VerificationPlan - obligations: list[VerificationObligation] - routing: list[RoutingDecision] - execution_records: list[ObligationExecutionRecord] - bundle: EvidenceBundle - policy_source: PolicySource -``` - -## Tests - -- `KernelResult.routing` IDs equal evidence routing IDs; -- denying a backend changes kernel execution; -- no second route is computed; -- CLI and MCP return the same selected/executed sets; -- routing digests remain identical through attestation; -- legacy mode is explicit and emits evidence v1 only; -- shadow mode contains both legacy and typed records with legacy authority clearly marked; -- enforced mode uses only typed control-plane results. - -## Sprint 3 exit criteria - -- one authoritative route exists per obligation; -- all public execution surfaces report the same route; -- no compatibility routing artifact is presented as authoritative. - -# 9. Sprint 4 — Hard execution isolation - -## Objective - -Ensure every authoritative adapter can be cancelled and cannot inherit ambient credentials or unrestricted filesystem access. - -## Files - -- `ovk/core/execution_budget.py` -- new `ovk/core/backend_workers.py` -- production adapters -- worker tests - -## Required worker implementations - -1. `LocalSubprocessWorker` -2. `SpawnedPythonWorker` -3. interface for later `ContainerWorker` - -## Required controls - -- wall-time timeout; -- process termination; -- bounded stdout and stderr; -- minimal environment; -- bounded working directory; -- explicit network policy metadata; -- explicit repository-write policy; -- process exit and signal recording; -- worker identity and image/runtime version; -- secret-pattern redaction before public artifact emission. - -## Deterministic adapters - -Move deterministic evaluators out of the control-plane process. - -Use a stable worker entry point such as: - -```text -python -I -m ovk.worker evaluate --adapter --input --output -``` - -The parent validates the worker output against a strict schema. - -## Tests - -- hanging deterministic adapter is terminated; -- child process attempting path escape is rejected; -- unknown parent credential is absent; -- explicit safe variable is available; -- oversized output is truncated and marked; -- malformed worker output becomes error; -- child process crash becomes error; -- total budget cancels remaining optional work; -- no completed result is emitted after timeout. - -## Sprint 4 exit criteria - -- no authoritative adapter executes unbounded in-process; -- every attempt has externally enforced termination semantics. - -# 10. Sprint 5 — Evidence v3 and cross-artifact binding - -## Objective - -Make the complete control-plane trace mandatory and verifiable by external consumers. - -## New schemas - -- `verification.evidence.v3.schema.json` -- `verification.bundle.v3.schema.json` -- `material.reference.schema.json` -- `material.set.schema.json` -- `execution.attempt.schema.json` -- strengthened routing and result schemas - -## Required evidence fields - -For `routing_enforced: true`, require: - -- obligation ID; -- routing ID; -- compiler identity; -- typed materials; -- material-set digest; -- typed coverage; -- requested candidates; -- eligible candidates with reasons; -- selected backends with required/optional roles; -- attempted backends; -- executed backends; -- execution attempts; -- guarantee classes; -- aggregation policy and reason; -- open obligations; -- policy digest; -- cache-hit state. - -## Canonical material-set digest - -Calculate over sorted canonical material references. - -Store and verify in: - -- obligation; -- evidence; -- provenance; -- attestation; -- release verifier. - -## Invariant upgrades - -Add canonical recomputation for: - -- obligation ID; -- routing ID; -- backend obligation ID; -- attempt ID; -- material-set digest; -- selected/executed set and role consistency; -- guarantee consistency; -- aggregate decision; -- attestation bindings. - -## Migration - -- read v1 and v2; -- generate v3 only for new enforced execution; -- provide migration documentation; -- do not silently convert v1/v2 to v3 without preserving absent-field warnings. - -## Sprint 5 exit criteria - -- an external validator can reconstruct every control-plane identity; -- artifacts fail when material, route, execution, or aggregate data disagree. - -# 11. Sprint 6 — Source profile hardening - -This sprint must proceed as separate profile-specific pull requests. - -## 11.1 FastAPI supported profile - -Replace regex-only route discovery with Python AST and module resolution. - -Support a documented subset: - -- `FastAPI` and `APIRouter` construction; -- decorator routes; -- `include_router`; -- static prefixes; -- application, router, and route dependencies; -- imported named dependencies; -- common role predicates; -- base/head reconstruction. - -Mark review for: - -- dynamic route registration; -- runtime factory-generated routers; -- unresolved imports; -- arbitrary metaprogramming; -- unsupported class dependencies. - -Coverage must count discovered application/router objects, include relationships, and route registrations. `expected_elements` cannot simply equal `extracted_elements`. - -## 11.2 Express supported profile - -Use TypeScript compiler API or ESTree-compatible parser. - -Support: - -- `express()`; -- `Router()`; -- static route registrations; -- middleware ordering; -- router mounts; -- import aliases; -- common auth and role middleware; -- base/head comparison. - -Mark review for dynamic paths, runtime registrations, unresolved module calls, and computed middleware arrays. - -## 11.3 Terraform supported profiles - -Use recursive plan traversal. - -Implement provider profiles separately, starting with a bounded set such as: - -- AWS S3 public exposure; -- AWS security-group/network path exposure; -- selected load-balancer entry points; -- one GCP storage profile or equivalent pilot requirement. - -Model: - -- `resource_changes` recursively; -- module addresses; -- `after_unknown`; -- sensitive values; -- provider defaults used by the profile; -- source addresses; -- policy and network relationships. - -## 11.4 Kubernetes supported profile - -Resolve: - -- namespace; -- Service selectors; -- workload labels; -- Ingress/Gateway backends; -- load-balancer address state; -- GatewayClass/IngressClass policy; -- NetworkPolicy intersections; -- relevant RBAC and ServiceAccount relationships. - -Do not classify every Ingress as public without profile evidence. - -## 11.5 GitHub Actions supported profile - -Implement: - -- workflow and job permission override semantics; -- event-specific secret availability; -- `pull_request_target` checkout trust; -- reusable workflow secret and permission propagation; -- `secrets: inherit`; -- local composite action expansion; -- protected environment configuration as trusted material; -- conservative `if:` evaluation; -- immutable versus mutable remote references. - -## 11.6 Deployment supported profiles - -Define strict profiles only over: - -- explicit OVK deployment schema; -- trusted GitHub Environment metadata; -- bounded Argo Rollouts fields. - -Artifact identity, approver evidence, promotion transitions, rollback reachability, and override authority must be typed materials. - -## Sprint 6 exit criteria - -Each source profile has: - -- a versioned support contract; -- positive corpus; -- negative corpus; -- unsupported corpus; -- independent annotations; -- source-range correctness tests; -- measurable coverage; -- documented strict eligibility. - -# 12. Sprint 7 — Template conformance v2 - -## Objective - -Replace repository-link existence with semantic conformance. - -## New status vocabulary - -- `catalog_only`; -- `executable_advisory`; -- `source_profile_strict_eligible`; -- `externally_calibrated_strict`; -- `deprecated`. - -## Required row fields - -Each template row must include: - -- compiler ID and version; -- supported source profile; -- material acquisition path; -- eligible backends; -- selected primary backend under test policy; -- guarantee class; -- pass fixture; -- fail fixture; -- malformed fixture; -- unknown fixture; -- timeout fixture; -- coverage fixture; -- counterexample validation; -- repair-artifact validation; -- evidence invariant references; -- package smoke reference; -- external calibration status; -- strict eligibility reason. - -## Required execution - -The conformance builder must run or consume machine-generated test results. File existence alone is insufficient. - -## Documentation - -README and status counts must derive from the generated conformance artifact. - -## Sprint 7 exit criteria - -- no template is marked strict eligible through path existence alone; -- every status is generated from tested semantic evidence. - -# 13. Sprint 8 — Holdout prediction pipeline - -## Objective - -Create an end-to-end, label-separated evaluation of the exact release-candidate artifact. - -## Job separation - -### Prediction job - -Receives: - -- case inputs without labels; -- exact release-candidate wheel or immutable commit; -- source SHA; -- wheel digest; -- no label access. - -Produces: - -- predictions; -- prediction digest; -- execution metadata; -- artifact signature or attestation. - -### Evaluation job - -Receives: - -- frozen holdout asset with independently recorded digest; -- predictions artifact and digest; -- label access; -- no release or repository credentials beyond asset retrieval. - -Produces only aggregate metrics. - -## Security requirements - -- network-disabled evaluator container where possible; -- read-only holdout materials; -- no GitHub token in evaluator process; -- path-safe extraction; -- immutable asset digest; -- output schema validation; -- aggregate leakage scan; -- no case IDs or labels in public artifacts. - -## Required aggregate provenance - -- holdout tag; -- holdout asset SHA-256; -- prediction artifact SHA-256; -- wheel SHA-256; -- OVK source SHA; -- workflow run ID; -- evaluator image digest; -- sanitizer version. - -## Sprint 8 exit criteria - -- predictions are generated by exact release-candidate code; -- labels never enter prediction job; -- public output contains aggregate metrics only; -- all identities and digests are retained. - -# 14. Sprint 9 — Independent consumer validation of current code - -## Objective - -Validate the new control-plane release candidate, not `v1.2.1`. - -## Repositories - -- `fraware/ovk-consumer-fastapi-terraform` -- `fraware/ovk-consumer-express-actions` - -## Required update - -Pin both to one of: - -- immutable release-candidate tag `v1.3.0-rc.1`; -- immutable audited commit SHA. - -Do not use `uses: ./`, `@main`, or `@master`. - -## Required workflow scenarios - -For each repository: - -1. advisory pass; -2. advisory block; -3. incomplete abstraction requiring review; -4. strict block; -5. backend unavailable; -6. backend timeout; -7. policy change; -8. cache miss and cache hit identity equivalence; -9. release bundle generation; -10. comment emission; -11. check-run emission; -12. true cross-fork PR with reduced permissions; -13. published or release-candidate wheel installation; -14. generated regression artifact; -15. source-profile compiler run. - -## Main-repository orchestration - -Replace pin-only validation with workflow dispatch and artifact ingestion. - -The main repository workflow must: - -- dispatch consumer workflow by immutable ref; -- await conclusion; -- verify exact OVK pin; -- download evidence artifacts; -- verify release bundles; -- confirm expected recommendation per scenario; -- write a machine-readable validation ledger. - -## Human pilot gate - -Automated fixtures do not satisfy the production pilot gate. - -Require per repository: - -- at least 30 human-adjudicated advisory PRs; -- false-positive records; -- missed-detection records; -- unknown appropriateness; -- reviewer burden; -- final merge disposition. - -## Sprint 9 exit criteria - -- both consumers validate the current release-candidate artifact; -- all automated scenarios are attributable; -- cross-fork permissions are exercised, not simulated only; -- human pilot ledgers remain clearly separate from automated fixtures. - -# 15. Sprint 10 — Release provenance and publication - -## Objective - -Create an attributable release candidate and later stable release. - -## Benchmark provenance correction - -Change generated benchmark fields: - -- `benchmark_source_sha` — source evaluated by benchmark workflow; -- `verified_source_sha` — source with complete required workflow evidence; -- `verified_workflow_runs` — required workflow names, run IDs, and conclusions. - -A benchmark-only workflow cannot populate `verified_source_sha`. - -## Release-candidate gates - -Require on the exact tag source: - -- general CI; -- native Tier 1; -- package and isolated wheel smoke; -- Action dogfood; -- release preflight; -- template conformance v2; -- expanded FormalPR-Bench; -- holdout prediction/evaluation; -- consumer validation; -- release-bundle verification; -- keyless Sigstore signing and verification; -- tamper tests. - -## Versioning - -Use: - -`v1.3.0-rc.1` - -until all P0 and automated validation gates pass. - -Promote to `v1.3.0` only after: - -- current-source required workflows are green; -- both consumer repositories pass the current pin; -- holdout aggregates are attributable; -- all P0 invariants are closed; -- release status is internally consistent. - -Keep package classifier Beta until human pilot thresholds and externally calibrated strict profiles are complete. - -# 16. Required pull-request decomposition - -Do not merge the remaining program as one monolithic change. - -Use the following PR order. - -## PR A — Deterministic attempt identity - -- remove duration from identity; -- add determinism tests; -- no behavior changes elsewhere. - -## PR B — Provenance-preserving cache v3 - -- cached execution model; -- cache schema bump; -- cache-hit provenance tests; -- no synthesized native state. - -## PR C — Coverage and guarantee contract enforcement - -- coverage-aware routing; -- guarantee mismatch failure; -- adversarial adapters and tests. - -## PR D — Fallback policy v2 - -- per-backend and per-guarantee fallback; -- timeout/error rejection; -- schema and migration. - -## PR E — Self-protection trust provenance - -- trust false by default; -- provenance model; -- protected-source tests. - -## PR F — Single authoritative routing pipeline - -- compile before route; -- one typed route; -- CLI/MCP parity; -- legacy API deprecation. - -## PR G — Isolated deterministic workers - -- spawned process worker; -- migrate one deterministic adapter; -- timeout and environment tests. - -## PR H — Remaining adapter isolation - -- migrate all authoritative adapters; -- resource and output bounds. - -## PR I — Evidence v3 and material-set binding - -- schemas; -- invariants; -- provenance and attestation; -- release verifier. - -## PR J — Template conformance v2 - -- status vocabulary; -- semantic fixtures; -- generated docs counts. - -Source compiler tracks may proceed in parallel after PR F, provided they use the centrally approved obligation, material, coverage, routing, and evidence models. - -# 17. Pull-request acceptance checklist - -Every PR must include: - -## Architecture - -- explicit interfaces; -- schema version impact; -- cache impact; -- migration impact; -- trust-boundary impact; -- guarantee statement; -- known limits. - -## Tests - -- passing case; -- failing case; -- malformed case; -- unknown case; -- timeout case where applicable; -- deterministic identity case; -- cache case where applicable; -- evidence-quality case; -- package or Action case where public behavior changes. - -## Security - -- no unrestricted path extraction; -- no unbounded inherited environment; -- no implicit fallback; -- no false native claim; -- no compiler guarantee relabeling; -- no untrusted metadata treated as authoritative by default; -- no PR-controlled policy governing its own enforcement without protected-source rules. - -## Artifacts - -- exact subject; -- material digests; -- compiler identity; -- coverage; -- selected and executed backend roles; -- execution provenance; -- assumptions and limits; -- attestation impact. - -## Documentation - -- status update; -- backend maturity update; -- template conformance update; -- migration note; -- release-gate impact. - -# 18. Definition of completed vision - -The current OVK vision is complete only when all conditions below hold. - -1. Every enforced obligation is routed exactly once. -2. `KernelResult`, evidence, provenance, and attestation reference the same routing ID. -3. Coverage requirements affect primary selection. -4. Compiler and routing guarantees cannot disagree silently. -5. Fallback is backend-, guarantee-, and cause-specific. -6. Cache hits preserve original execution provenance. -7. Attempt and bundle identities are stable across equivalent runs. -8. Every authoritative adapter is externally time-bounded. -9. Self-protection trust is explicit and protected-source-bound. -10. Evidence schema requires the complete control-plane trace. -11. Material-set digest is cross-verified across all release artifacts. -12. Strict-eligible templates pass semantic conformance, not path-existence checks. -13. Source profiles have measurable completeness and unsupported-case behavior. -14. The exact current source passes all required workflows. -15. The exact current wheel and Action pass both consumer repositories. -16. The exact current source generates holdout predictions without labels. -17. Aggregate holdout and consumer evidence is retained with immutable digests. -18. Public release claims match these artifacts. - -# 19. Immediate engineer assignments - -Start with these five assignments. - -## Assignment 1 — Kernel identity and cache team - -Deliver PR A and PR B. - -Do not modify routing policy in these PRs. - -## Assignment 2 — Router and aggregation team - -Deliver PR C and PR D. - -Use defective adapters in adversarial tests. - -## Assignment 3 — Trust-boundary team - -Deliver PR E and validate the worker and holdout corrections already committed during this audit. - -## Assignment 4 — Kernel integration team - -Design PR F after PR A–E interfaces stabilize. - -Produce a design note showing how compatibility routing is removed from the authoritative execution path. - -## Assignment 5 — Release and external validation team - -- run the current-source baseline; -- correct benchmark source/verified source terminology; -- prepare `v1.3.0-rc.1` consumer pins; -- orchestrate consumer workflows; -- implement label-separated holdout prediction. - -# 20. Final instruction - -The project should now optimize for semantic integrity, not breadth. - -Do not add another backend, template family, or marketing claim until: - -- cached provenance is correct; -- attempt identity is deterministic; -- coverage governs selection; -- fallback is narrowly enforced; -- routing is unified; -- self-protection trust is fail-closed; -- current-source external validation exists. - -The architecture is now sufficiently ambitious. The next engineering quality threshold is proving that every identity, guarantee, material, execution, and release claim remains correct under adversarial conditions. +Author this document and the deep audit; rewrite [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) for `v1.3.0-rc.1`; land CI-secrets `size_bytes`, worker env allowlist + zero-budget reject, and FormalPR-Holdout isolation hotfixes. diff --git a/docs/EXPERIMENTAL_PATHS.md b/docs/EXPERIMENTAL_PATHS.md index fb284a8..61ece26 100644 --- a/docs/EXPERIMENTAL_PATHS.md +++ b/docs/EXPERIMENTAL_PATHS.md @@ -56,6 +56,8 @@ mismatches are cache misses — stale results must not authorize allow. ## Metric provenance -Generated badge/summary/adoption metrics must carry `verified_source_sha` for -the commit that produced the numbers. Later `[skip ci]` badge commits must not -be cited as the verified source. See program §24.3. +Generated badge/summary/adoption metrics must carry `benchmark_source_sha` for +the commit that produced the numbers. Set `verified_source_sha` only when a +complete required-workflow set was observed. Later `[skip ci]` badge commits must +not be cited as the verified source. See program §24.3 and +[CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). diff --git a/docs/FORMALPR_HOLDOUT_GOVERNANCE.md b/docs/FORMALPR_HOLDOUT_GOVERNANCE.md index 5309802..bd73665 100644 --- a/docs/FORMALPR_HOLDOUT_GOVERNANCE.md +++ b/docs/FORMALPR_HOLDOUT_GOVERNANCE.md @@ -35,7 +35,9 @@ Protected labels live only in private release assets (and gitignored local - Publishes **aggregate metrics only**; fails closed if labels or case ids would be printed. - Ordinary `CI` jobs do **not** checkout holdout labels. -Runner: `scripts/run_formalpr_holdout.py`. +Runner: `scripts/run_formalpr_holdout.py` (requires immutable `--asset-sha256`). + +Label-separated prediction/eval flow (Sprint 8): [HOLDOUT_LABEL_SEPARATION.md](HOLDOUT_LABEL_SEPARATION.md). ## What this is not diff --git a/docs/HOLDOUT_LABEL_SEPARATION.md b/docs/HOLDOUT_LABEL_SEPARATION.md new file mode 100644 index 0000000..e154750 --- /dev/null +++ b/docs/HOLDOUT_LABEL_SEPARATION.md @@ -0,0 +1,35 @@ +# Label-Separated Holdout Evaluation (Sprint 8) + +Checklist for R2 Sprint 8. Builds on Phase A FormalPR-Holdout isolation +(`scripts/run_formalpr_holdout.py`, `.github/workflows/holdout-eval.yml`). + +## Required flow + +1. **Predict** using the exact RC artifact (wheel / Action pin) **without** access to protected labels. +2. **Digest** the predictions file (`scripts/digest_holdout_predictions.py`) — refuses embedded labels / ground-truth fields; emits SHA-256 record. +3. **Evaluate separately** with protected labels (token only on download step; evaluator env token-free). +4. **Publish aggregates only** (`formalpr_holdout.aggregate_metrics.v1`). + +## In-repo artifacts + +| Item | Path / note | +|---|---| +| Runner | `scripts/run_formalpr_holdout.py` (requires `--asset-sha256`; validates predictions are label-free) | +| Predictions digest | `scripts/digest_holdout_predictions.py` | +| Workflow | `.github/workflows/holdout-eval.yml` (download vs eval token split) | +| Predictions placeholder | `.verification/holdout-predictions.json` (never commit labels) | +| Governance | [FORMALPR_HOLDOUT_GOVERNANCE.md](FORMALPR_HOLDOUT_GOVERNANCE.md) | + +## Checklist + +- [ ] RC predictions generated in an environment without `corpus/labels` +- [ ] Predictions digested (`digest_holdout_predictions.py`) and retained with workflow ID +- [ ] `HOLDOUT_ASSET_SHA256` (or workflow input) set for immutable asset verify +- [ ] Eval job runs with tokens unset; aggregates schema-validated +- [ ] Published metrics cite `ovk_commit_sha` / `benchmark_source_sha` and do not embed case ids +- [ ] Do not set `verified_source_sha` on holdout aggregates unless the full required-workflow set was observed + +## Blocked outside this repo + +Protected label store and annotator workflow live in `fraware/FormalPR-Holdout` (private). +This repository cannot complete live holdout scoring without that access. diff --git a/docs/README.md b/docs/README.md index 805c982..5b95b54 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # OVK Documentation -Documentation for Open Verification Kernel **v1.2.0**. +Documentation for Open Verification Kernel (**v1.3.0-rc.1 candidate**; signed tag remains `v1.2.1`). Use this index as the canonical entry point. Each guide covers one topic; cross-links replace duplicated content across files. @@ -11,13 +11,15 @@ Use this index as the canonical entry point. Each guide covers one topic; cross- | **Adopting in CI** | [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) → [INTEGRATION.md](INTEGRATION.md) → [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) | | **Contributing code** | [CONTRIBUTING.md](CONTRIBUTING.md) → [ARCHITECTURE.md](ARCHITECTURE.md) → [ADAPTER_CONTRACT.md](ADAPTER_CONTRACT.md) | | **Maintainers** | [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) → [RELEASE.md](RELEASE.md) → `ovk release-preflight` (release readiness checks) | -| **Spec / security review** | [SYSTEM_SPEC.md](SYSTEM_SPEC.md) → [FORMAL_SPEC.md](FORMAL_SPEC.md) → [THREAT_MODEL.md](THREAT_MODEL.md) | +| **Spec / security review** | [SYSTEM_SPEC.md](SYSTEM_SPEC.md) → [FORMAL_SPEC.md](FORMAL_SPEC.md) → [THREAT_MODEL.md](THREAT_MODEL.md) → [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | ## Start here | Document | Purpose | |---|---| | [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) | Adoption dashboard — can I pin strict mode today? | +| [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | Authoritative R2 deep audit (supersedes day-to-day VISION_AUDIT) | +| [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md) | R2 sprint/PR execution program | | [RELEASE_AUDIT.md](RELEASE_AUDIT.md) | Engineering audit responses and metric provenance | | [STATUS.md](STATUS.md) | Capabilities, CLI surface, and trust model | | [EXPERIMENTAL_PATHS.md](EXPERIMENTAL_PATHS.md) | Honest limits for non-strict compiler/backend paths | @@ -35,6 +37,9 @@ Use this index as the canonical entry point. Each guide covers one topic; cross- | [EXTERNAL_VALIDATION.md](EXTERNAL_VALIDATION.md) | Weekly external validation matrix | | [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) | Immutable-pin consumer validation checklist (scaffolding) | | [FORMALPR_HOLDOUT_GOVERNANCE.md](FORMALPR_HOLDOUT_GOVERNANCE.md) | Private FormalPR-Holdout governance stub (no corpus) | +| [HOLDOUT_LABEL_SEPARATION.md](HOLDOUT_LABEL_SEPARATION.md) | Sprint 8 label-separated prediction/eval checklist | +| [SOURCE_PROFILE_HARDENING.md](SOURCE_PROFILE_HARDENING.md) | Sprint 6 source-profile hardening scaffolding | +| [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md) | Sprint 10 rc.1 / v1.3.0 publication gate | | [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) | Advisory→strict rollout on external OSS repos | | [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md) | In-repo pilot metrics and external pilot reporting | | [AGENT_REPAIR_LOOP.md](AGENT_REPAIR_LOOP.md) | Counterexample-to-repair workflow for MCP agents | diff --git a/docs/RELEASE.md b/docs/RELEASE.md index f88080d..8ce747d 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -1,6 +1,6 @@ # OVK Release -Maintainer guide for shipping Open Verification Kernel. Current readiness: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). Full code and artifact assessment: [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md). +Maintainer guide for shipping Open Verification Kernel. Current readiness: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). Authoritative audit: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Engineering program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). Historical: [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md). ## Current release candidate diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 341dc7a..9ff6e34 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,23 +1,26 @@ # OVK Roadmap -Current release: **v1.2.0**. What OVK can do today: [STATUS.md](STATUS.md). Adoption status: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). +Current product positioning: **`v1.3.0-rc.1` candidate** (typed control plane post-dates signed `v1.2.1`). What OVK can do today: [STATUS.md](STATUS.md). Adoption status: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). Authoritative program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). ## Release history | Version | Summary | Changelog | |---|---|---| +| v1.3.0-rc.1 (candidate) | Typed backend control plane; P0 trust PRs 1–9 in working tree; publication gates open | [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | +| v1.2.1 | Signed release on pre-control-plane commit; consumer pin baseline | [RELEASE_NOTES_v1.2.1.md](RELEASE_NOTES_v1.2.1.md) | | v1.2.0 | All five check types validated end-to-end; clearer GitHub Action outputs; example rollout workflows | [RELEASE_NOTES_v1.2.0.md](RELEASE_NOTES_v1.2.0.md) | | v1.1.0 | Realistic PR diff benchmark set; required native checker CI for OPA, Z3, CBMC, Cedar; external rollout guide | [RELEASE_NOTES_v1.1.0.md](RELEASE_NOTES_v1.1.0.md) | | v1.0.0 | Unified `ovk check`, five check types, ten backends, GitHub Action, benchmark suite | [RELEASE_NOTES_v1.0.0.md](RELEASE_NOTES_v1.0.0.md) | ## What we are working on next -1. **External repository pilots** — publish measured outcomes from real open-source adopters in [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md). -2. **Richer authorization checks** — more expressive Z3 obligations and smaller, clearer counterexamples. -3. **Community backends** — adapters and install docs beyond the current native checker matrix. -4. **CBMC diff depth** — richer extraction from real C PR hunks and repair-loop fixtures for CBMC counterexamples. +1. **Sprint 0 / attributable gates** — live CI, wheel smoke, Action dogfood, and workflow IDs on a non-`[skip ci]` SHA (P0 code PRs 1–9 already in working tree; see [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md)). +2. **Semantic template conformance v2** and source-profile hardening (Sprints 6–7). +3. **Consumer validation on rc.1** and label-separated holdout (Sprints 8–9). +4. **Attributable publication** of `v1.3.0-rc.1` then `v1.3.0` after the 18-condition gate (Sprint 10). ## Not planned as product promises - PyPI publication depends on maintainer release tagging (workflow is ready). - Optional native checkers (TLA+, Kani, Dafny, Verus, Lean, Alloy) remain non-blocking in CI until their harnesses mature. +- Re-attributing `v1.2.1` Sigstore/CI evidence to typed-control-plane commits. diff --git a/docs/SOURCE_PROFILE_HARDENING.md b/docs/SOURCE_PROFILE_HARDENING.md new file mode 100644 index 0000000..643a75a --- /dev/null +++ b/docs/SOURCE_PROFILE_HARDENING.md @@ -0,0 +1,48 @@ +# Source Profile Hardening (Sprint 6) + +Hardening beyond scaffolding. Authoritative program: +[ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). + +## Goals + +Replace regex-only / heuristic extraction with explicit **source profiles** that +authorize deeper analysis only when trusted materials are present: + +| Lane | Hardening target | +|---|---| +| Authorization | AST / module-graph profiles (FastAPI, Express) | +| Infrastructure | Recursive Terraform plan expansion; controller-aware Kubernetes reachability | +| CI secrets | Deeper Actions permissions and secret-flow modeling | +| Deployment | Strictness only on explicit trusted profiles | + +## Status in this tree + +| Profile | Implementation | +|---|---| +| `authorization.fastapi.ast_v1` | `FastApiAstAuthorizationCompiler` (Python AST; preferred over regex) | +| `authorization.express.ast_v1` | Express compiler still regex/module-import based; deeper module-graph pending | +| `infrastructure.terraform.plan_recursive_v1` | Recursive `child_modules` walk in `compile_terraform_plan` | +| `infrastructure.kubernetes.controller_reachability_v1` | Service selector edges to Deployment/StatefulSet/DaemonSet | +| `ci_secrets.actions.permissions_flow_v1` | Permissions + secret extraction via `compile_workflow_trust` | +| `deployment.trusted_profile_v1` | Strictness gated on explicit `trusted_profile.v1.json` material | + +Evidence collection for template conformance v2 runs these provers from +`ovk/core/source_profile_evidence.py`. Profile IDs and compiler bindings live in +`ovk/core/source_profiles.py`. + +## Remaining gaps (honest) + +- Express AST / full module-graph resolution is not yet equivalent to FastAPI AST. +- Actions composite/reusable recursion beyond current trust-flow expansion. +- Deployment does **not** claim `source_profile_strict_eligible` until an explicit + trusted profile material is present under `examples/deployment_state/`. +- `externally_calibrated_strict` is never granted by local generation alone. + +## Gate + +`source_profile_strict_eligible` requires: + +1. profile ID recorded on the obligation / evidence; +2. materials marked trusted with matching profile provenance; +3. coverage status `complete` for the profile's extracted elements; +4. enforcement test covering the profile path. diff --git a/docs/VISION_AUDIT_2026-07-22.md b/docs/VISION_AUDIT_2026-07-22.md index 0bf35b5..b9fcd55 100644 --- a/docs/VISION_AUDIT_2026-07-22.md +++ b/docs/VISION_AUDIT_2026-07-22.md @@ -1,5 +1,7 @@ # OVK Vision Audit — 2026-07-22 +> **Historical.** For current release judgment and P0 trust gaps, use [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) and [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). This document predates the typed backend control plane and describes routing as advisory metadata. + ## Executive judgment Open Verification Kernel has achieved a credible **verification evidence product** for a bounded set of high-risk pull-request changes. It can infer five core checks from diffs, compile normalized lane inputs, execute deterministic or selected native checkers, aggregate evidence, produce conservative merge recommendations, render review output, and write hash-bound release bundles. diff --git a/docs/benchmarks/template-conformance.json b/docs/benchmarks/template-conformance.json index d7963a8..e5186c1 100644 --- a/docs/benchmarks/template-conformance.json +++ b/docs/benchmarks/template-conformance.json @@ -1,4 +1,11 @@ { + "conformance_statuses_v2": [ + "deprecated", + "catalog_only", + "executable_advisory", + "source_profile_strict_eligible", + "externally_calibrated_strict" + ], "counts_by_domain": { "agent_authority": 12, "authorization": 18, @@ -11,6 +18,11 @@ "catalog_only": 95, "strict_eligible": 5 }, + "counts_by_status_v2": { + "catalog_only": 95, + "executable_advisory": 2, + "source_profile_strict_eligible": 3 + }, "production_statuses": [ "deprecated", "catalog_only", @@ -43,6 +55,53 @@ "notes" ], "schema_version": "ovk.template_conformance.v1", + "source_profile_evidence": { + "no-admin-route-bypass": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "routes=1", + "profile_marker=yes" + ], + "profile_id": "authorization.fastapi.ast_v1", + "strict_eligible": true + }, + "no-public-sensitive-resource": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "resources=1", + "eligibility=strict" + ], + "profile_id": "infrastructure.terraform.plan_recursive_v1", + "strict_eligible": true + }, + "no-secrets-in-untrusted-context": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "secrets=2", + "write_token=True", + "findings=2", + "compiled_with_source_profile:ci_secrets.actions.permissions_flow_v1" + ], + "profile_id": "ci_secrets.actions.permissions_flow_v1", + "strict_eligible": true + }, + "no-skipped-approval-state": { + "coverage_complete": false, + "enforcement_test_present": true, + "materials_trusted": false, + "notes": [ + "requires explicit trusted_profile material for strictness" + ], + "profile_id": "deployment.trusted_profile_v1", + "strict_eligible": false + } + }, "template_count": 100, "templates": [ { @@ -51,6 +110,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -86,6 +146,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -121,6 +182,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -153,6 +215,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -185,6 +248,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -217,6 +281,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -249,6 +314,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -281,6 +347,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -313,6 +380,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -345,6 +413,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -381,6 +450,7 @@ "claimed_backends": [ "dafny" ], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -418,6 +488,7 @@ "claimed_backends": [ "opa" ], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -454,6 +525,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -489,6 +561,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -524,6 +597,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -560,6 +634,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -598,6 +673,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -635,6 +711,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -673,6 +750,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -711,6 +789,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -751,6 +830,7 @@ "authorization-deterministic", "z3-native" ], + "conformance_status_v2": "source_profile_strict_eligible", "domain": "authorization", "executable_links": { "backend_registry": true, @@ -765,12 +845,25 @@ "lane": "authorization", "missing_executable_links": [], "notes": [ - "all required executable links present" + "all required executable links present", + "source_profile_evidence:strict_ok" ], "path": "templates/authorization/no_admin_route_bypass.intent.json", "production_status": "strict_eligible", "property_kind": "access_control", "risk_severity": "high", + "source_profile_evidence": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "routes=1", + "profile_marker=yes" + ], + "profile_id": "authorization.fastapi.ast_v1", + "strict_eligible": true + }, + "source_profile_id": "authorization.fastapi.ast_v1", "version": "0.1.0" }, { @@ -780,6 +873,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -813,6 +907,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -845,6 +940,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -877,6 +973,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -909,6 +1006,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -941,6 +1039,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -973,6 +1072,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -1005,6 +1105,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -1042,6 +1143,7 @@ "claimed_backends": [ "kani" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -1081,6 +1183,7 @@ "opa-native", "self-protection-deterministic" ], + "conformance_status_v2": "executable_advisory", "domain": "ci_cd", "executable_links": { "backend_registry": true, @@ -1109,6 +1212,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1144,6 +1248,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1179,6 +1284,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1214,6 +1320,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1246,6 +1353,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1278,6 +1386,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1310,6 +1419,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1342,6 +1452,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1374,6 +1485,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1406,6 +1518,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1438,6 +1551,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1470,6 +1584,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1507,6 +1622,7 @@ "claimed_backends": [ "kani" ], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1545,6 +1661,7 @@ "claimed_backends": [ "kani" ], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1583,6 +1700,7 @@ "claimed_backends": [ "kani" ], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1618,6 +1736,7 @@ "claimed_backends": [ "ci-secrets-deterministic" ], + "conformance_status_v2": "source_profile_strict_eligible", "domain": "ci_cd", "executable_links": { "backend_registry": true, @@ -1632,12 +1751,27 @@ "lane": "ci_secrets", "missing_executable_links": [], "notes": [ - "all required executable links present" + "all required executable links present", + "source_profile_evidence:strict_ok" ], "path": "templates/ci_cd/no_secrets_in_untrusted_context.intent.json", "production_status": "strict_eligible", "property_kind": "data_boundary", "risk_severity": "critical", + "source_profile_evidence": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "secrets=2", + "write_token=True", + "findings=2", + "compiled_with_source_profile:ci_secrets.actions.permissions_flow_v1" + ], + "profile_id": "ci_secrets.actions.permissions_flow_v1", + "strict_eligible": true + }, + "source_profile_id": "ci_secrets.actions.permissions_flow_v1", "version": "0.1.0" }, { @@ -1647,6 +1781,7 @@ "claimed_backends": [ "verus" ], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1684,6 +1819,7 @@ "claimed_backends": [ "z3" ], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1721,6 +1857,7 @@ "claimed_backends": [ "cbmc" ], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1759,6 +1896,7 @@ "claimed_backends": [ "cbmc" ], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1797,6 +1935,7 @@ "claimed_backends": [ "cbmc" ], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1835,6 +1974,7 @@ "claimed_backends": [ "cbmc" ], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1871,6 +2011,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1906,6 +2047,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1941,6 +2083,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1973,6 +2116,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2005,6 +2149,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2037,6 +2182,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2069,6 +2215,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2101,6 +2248,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2133,6 +2281,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2165,6 +2314,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2201,6 +2351,7 @@ "claimed_backends": [ "lean" ], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2234,6 +2385,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2266,6 +2418,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2298,6 +2451,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2330,6 +2484,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2362,6 +2517,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2394,6 +2550,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2426,6 +2583,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2458,6 +2616,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2493,6 +2652,7 @@ "trace" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2528,6 +2688,7 @@ "trace" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2563,6 +2724,7 @@ "trace" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2597,6 +2759,7 @@ "claimed_backends": [ "deployment-deterministic" ], + "conformance_status_v2": "executable_advisory", "domain": "deployment", "executable_links": { "backend_registry": true, @@ -2611,12 +2774,24 @@ "lane": "deployment", "missing_executable_links": [], "notes": [ - "all required executable links present" + "all required executable links present", + "source_profile_evidence:incomplete" ], "path": "templates/deployment/no_skipped_approval_state.intent.json", "production_status": "strict_eligible", "property_kind": "invariant", "risk_severity": "high", + "source_profile_evidence": { + "coverage_complete": false, + "enforcement_test_present": true, + "materials_trusted": false, + "notes": [ + "requires explicit trusted_profile material for strictness" + ], + "profile_id": "deployment.trusted_profile_v1", + "strict_eligible": false + }, + "source_profile_id": "deployment.trusted_profile_v1", "version": "0.1.0" }, { @@ -2626,6 +2801,7 @@ "claimed_backends": [ "tla" ], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2664,6 +2840,7 @@ "claimed_backends": [ "tla" ], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2702,6 +2879,7 @@ "claimed_backends": [ "tla" ], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2739,6 +2917,7 @@ "claimed_backends": [ "tla" ], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2777,6 +2956,7 @@ "claimed_backends": [ "tla" ], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2815,6 +2995,7 @@ "claimed_backends": [ "alloy" ], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -2853,6 +3034,7 @@ "claimed_backends": [ "alloy" ], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -2891,6 +3073,7 @@ "claimed_backends": [ "alloy" ], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -2928,6 +3111,7 @@ "claimed_backends": [ "alloy" ], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -2961,6 +3145,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -2993,6 +3178,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3025,6 +3211,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3057,6 +3244,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3089,6 +3277,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3121,6 +3310,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3153,6 +3343,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3185,6 +3376,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3220,6 +3412,7 @@ "topology_model" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3255,6 +3448,7 @@ "topology_model" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3290,6 +3484,7 @@ "topology_model" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3325,6 +3520,7 @@ "topology_model" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3359,6 +3555,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3395,6 +3592,7 @@ "claimed_backends": [ "alloy" ], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3433,6 +3631,7 @@ "claimed_backends": [ "infrastructure-deterministic" ], + "conformance_status_v2": "source_profile_strict_eligible", "domain": "infrastructure", "executable_links": { "backend_registry": true, @@ -3447,12 +3646,25 @@ "lane": "infrastructure", "missing_executable_links": [], "notes": [ - "all required executable links present" + "all required executable links present", + "source_profile_evidence:strict_ok" ], "path": "templates/infrastructure/no_public_sensitive_resource.intent.json", "production_status": "strict_eligible", "property_kind": "data_boundary", "risk_severity": "high", + "source_profile_evidence": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "resources=1", + "eligibility=strict" + ], + "profile_id": "infrastructure.terraform.plan_recursive_v1", + "strict_eligible": true + }, + "source_profile_id": "infrastructure.terraform.plan_recursive_v1", "version": "0.1.0" } ] diff --git a/docs/templates/consumer_validation.workflow.yml b/docs/templates/consumer_validation.workflow.yml index a2ac0fc..3bdb389 100644 --- a/docs/templates/consumer_validation.workflow.yml +++ b/docs/templates/consumer_validation.workflow.yml @@ -4,10 +4,15 @@ name: OVK Consumer Validation # Live examples: # https://github.com/fraware/ovk-consumer-fastapi-terraform # https://github.com/fraware/ovk-consumer-express-actions +# +# Sprint 9 target pin: immutable v1.3.0-rc.1 (or audited full SHA) AFTER the +# attributable rc.1 tag exists. Until then, keep the signed v1.2.1 pin. +# Do not push consumer changes from this template alone. on: pull_request: branches: [main] + workflow_dispatch: permissions: contents: read @@ -15,7 +20,9 @@ permissions: checks: write env: - OVK_PACKAGE_VERSION: "1.2.1" + # After rc.1 cut: "1.3.0rc1". Until then keep "1.2.1". + OVK_PACKAGE_VERSION: "1.3.0rc1" + OVK_ACTION_REF: "v1.3.0-rc.1" jobs: ovk-consumer-validation: @@ -24,8 +31,14 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 2 + - name: Guard pin target + run: | + echo "Expected Action ref: ${OVK_ACTION_REF}" + echo "Expected package: ${OVK_PACKAGE_VERSION}" + echo "Refuse floating @main pins. Prefer tag or full commit SHA." - name: Run OVK advisory consumer path - uses: fraware/open-verification-kernel@v1.2.1 + # Replace with the immutable tag/SHA once v1.3.0-rc.1 exists. + uses: fraware/open-verification-kernel@v1.3.0-rc.1 with: mode: advisory use-check: "true" @@ -35,3 +48,4 @@ jobs: run: | echo "Adjudicate this PR into a pilot ledger conforming to schemas/pilot.ledger.schema.json" echo "Do not set production_gate_met=true until 30 human adjudications exist." + echo "Keep automated_scenario rows distinct from human adjudications." diff --git a/ovk/adapters/authorization/deterministic_adapter.py b/ovk/adapters/authorization/deterministic_adapter.py index 39b7508..ef20049 100644 --- a/ovk/adapters/authorization/deterministic_adapter.py +++ b/ovk/adapters/authorization/deterministic_adapter.py @@ -2,13 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.z3.counterexample import counterexamples_from_obligation -from ovk.adapters.z3.obligation import build_authorization_obligation -from ovk.adapters.z3.validation import validate_authorization_input from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -26,9 +22,10 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -142,9 +139,7 @@ def compile( environment_requirements={"native": False}, expected_guarantee="deterministic_witness", ) - return provisional.model_copy( - update={"backend_obligation_id": compute_backend_obligation_id(provisional)} - ) + return provisional.model_copy(update={"backend_obligation_id": compute_backend_obligation_id(provisional)}) def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironmentFingerprint: return BackendEnvironmentFingerprint( @@ -161,55 +156,18 @@ def run( self, backend_obligation: BackendObligation, budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, ) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - data = dict(backend_obligation.payload.get("input") or {}) - issues = validate_authorization_input(data) - if issues: - raw_result = { - "status": "unknown", - "reason": "malformed authorization input", - "issues": issues, - "models": [], - } - termination = "invalid_output" - else: - auth_obligation = build_authorization_obligation(data) - counterexamples = counterexamples_from_obligation(auth_obligation) - raw_result = { - "status": "fail" if counterexamples else "pass", - "reason": ( - "deterministic violation witness found" - if counterexamples - else "no deterministic violation witness found" - ), - "models": counterexamples, - "counterexamples": counterexamples, - } - termination = "completed" - # Soft timeout signal for tests: budget of 0 forces timeout. - duration_ms = (time.perf_counter() - started) * 1000.0 - if budget.per_backend_wall_time_seconds <= 0: - termination = "timeout" - raw_result = { - "status": "unknown", - "reason": "budget timeout", - "models": [], - } - raw = RawBackendExecution( + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination=termination, # type: ignore[arg-type] - native_execution=False, - exit_code=0 if termination == "completed" else 1, - raw_result=raw_result, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=duration_ms, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="authorization-deterministic", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, diff --git a/ovk/adapters/authorization/z3_adapter.py b/ovk/adapters/authorization/z3_adapter.py index 9ad743c..c26a81e 100644 --- a/ovk/adapters/authorization/z3_adapter.py +++ b/ovk/adapters/authorization/z3_adapter.py @@ -2,14 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.z3.executor import run_authorization_obligation_with_z3 -from ovk.adapters.z3.obligation import build_authorization_obligation -from ovk.adapters.z3.result import normalize_z3_authorization_result -from ovk.adapters.z3.validation import validate_authorization_input from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -27,9 +22,10 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -160,9 +156,7 @@ def compile( environment_requirements={"native": True, "binary": "z3-solver"}, expected_guarantee="smt_refutation_search", ) - return provisional.model_copy( - update={"backend_obligation_id": compute_backend_obligation_id(provisional)} - ) + return provisional.model_copy(update={"backend_obligation_id": compute_backend_obligation_id(provisional)}) def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironmentFingerprint: native = z3_available() @@ -193,82 +187,18 @@ def run( self, backend_obligation: BackendObligation, budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, ) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - data = dict(backend_obligation.payload.get("input") or {}) - if budget.per_backend_wall_time_seconds <= 0: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="timeout", - native_execution=True, - exit_code=1, - raw_result={"status": "unknown", "reason": "budget timeout", "models": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - issues = validate_authorization_input(data) - if issues: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="invalid_output", - native_execution=False, - exit_code=1, - raw_result={"status": "unknown", "reason": "malformed input", "issues": issues, "models": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - if not z3_available(): - # Explicit non-fallback: native adapter reports unavailable/unknown. - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="tool_unavailable", - native_execution=False, - exit_code=1, - raw_result={ - "status": "unknown", - "reason": "z3-solver is not installed", - "models": [], - }, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - auth_obligation = build_authorization_obligation(data) - native_raw = run_authorization_obligation_with_z3(auth_obligation) - normalized = normalize_z3_authorization_result(native_raw) - raw = RawBackendExecution( + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination="completed", - native_execution=True, - exit_code=0, - raw_result={ - "status": normalized["status"], - "reason": native_raw.get("reason"), - "models": native_raw.get("models", []), - "counterexamples": normalized.get("counterexamples", []), - }, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="z3-authorization-native", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, diff --git a/ovk/adapters/cbmc/diff_extract.py b/ovk/adapters/cbmc/diff_extract.py index 5df2b7c..770e01c 100644 --- a/ovk/adapters/cbmc/diff_extract.py +++ b/ovk/adapters/cbmc/diff_extract.py @@ -23,9 +23,7 @@ def _findings_from_c(content: str, *, intent_id: str) -> list[dict[str, str]]: findings: list[dict[str, str]] = [] if intent_id == "cbmc-no-integer-overflow-quota" and re.search(r"\+\s*=|\+\+|quota|limit", content): findings.append({"kind": "integer_overflow", "summary": "quota or counter arithmetic changed in diff"}) - if intent_id == "cbmc-no-unchecked-buffer-copy" and re.search( - r"\b(memcpy|strcpy|strncpy|memmove)\s*\(", content - ): + if intent_id == "cbmc-no-unchecked-buffer-copy" and re.search(r"\b(memcpy|strcpy|strncpy|memmove)\s*\(", content): findings.append({"kind": "unchecked_copy", "summary": "memory copy operation introduced or modified in diff"}) if intent_id == "cbmc-no-use-after-free-auth-cache" and re.search(r"\bfree\s*\(", content): findings.append({"kind": "use_after_free", "summary": "free() call introduced or modified in diff"}) diff --git a/ovk/adapters/cbmc/harness_compiler.py b/ovk/adapters/cbmc/harness_compiler.py index 2e1a751..71ffbd6 100644 --- a/ovk/adapters/cbmc/harness_compiler.py +++ b/ovk/adapters/cbmc/harness_compiler.py @@ -74,10 +74,7 @@ def _generated_integer_overflow_harness(data: dict[str, Any]) -> str: expect_violation = bool(data.get("expect_violation", data.get("failed_assertions"))) guard = "" if not expect_violation: - guard = ( - f" __CPROVER_assume(used <= {quota_limit});\n" - f" __CPROVER_assume(delta <= {quota_limit} - used);\n" - ) + guard = f" __CPROVER_assume(used <= {quota_limit});\n __CPROVER_assume(delta <= {quota_limit} - used);\n" return f"""#include #include diff --git a/ovk/adapters/cedar/adapter.py b/ovk/adapters/cedar/adapter.py index 968386e..fc530b6 100644 --- a/ovk/adapters/cedar/adapter.py +++ b/ovk/adapters/cedar/adapter.py @@ -58,7 +58,9 @@ def evaluate_evidence( "The decision is produced by the deterministic Cedar-shaped input oracle.", ] if binary_present: - assumptions.append("The Cedar CLI version probe passed; no policy evaluation was executed by the native tool.") + assumptions.append( + "The Cedar CLI version probe passed; no policy evaluation was executed by the native tool." + ) else: assumptions.append("The Cedar CLI was unavailable; native policy evaluation was not attempted.") diff --git a/ovk/adapters/ci_secrets/deterministic_adapter.py b/ovk/adapters/ci_secrets/deterministic_adapter.py index 63ed85f..b1e3659 100644 --- a/ovk/adapters/ci_secrets/deterministic_adapter.py +++ b/ovk/adapters/ci_secrets/deterministic_adapter.py @@ -2,11 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.ci_secrets.exposure import find_ci_secrets_counterexamples from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -24,9 +22,10 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -154,52 +153,22 @@ def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironme native_available=False, ) - def run(self, backend_obligation: BackendObligation, budget: ExecutionBudget) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - if budget.per_backend_wall_time_seconds <= 0: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="timeout", - native_execution=False, - exit_code=1, - raw_result={"status": "unknown", "reason": "budget timeout", "counterexamples": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - data = dict(backend_obligation.payload.get("input") or {}) - workflows = data.get("workflows") - if not isinstance(workflows, list) or not workflows: - status = "unknown" - counterexamples = [ - { - "summary": "Workflow abstraction is missing or empty.", - "failure_mode": "missing_workflow_abstraction", - } - ] - termination = "invalid_output" - else: - counterexamples = find_ci_secrets_counterexamples(data) - status = "fail" if counterexamples else "pass" - termination = "completed" - raw = RawBackendExecution( + def run( + self, + backend_obligation: BackendObligation, + budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, + ) -> RawBackendExecution: + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination=termination, # type: ignore[arg-type] - native_execution=False, - exit_code=0 if termination == "completed" else 1, - raw_result={"status": status, "counterexamples": counterexamples}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="ci-secrets-deterministic", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, @@ -221,9 +190,7 @@ def normalize( assumptions=["Deterministic CI secrets boundary evaluator."], limits=["Does not expand remote reusable workflows."], counterexamples=list(raw.raw_result.get("counterexamples") or []), - generated_artifacts=[ - {"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False} - ], + generated_artifacts=[{"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False}], ) def explain(self, result: NormalizedBackendResult) -> HumanExplanation: diff --git a/ovk/adapters/ci_secrets/exposure.py b/ovk/adapters/ci_secrets/exposure.py index 699156f..960a219 100644 --- a/ovk/adapters/ci_secrets/exposure.py +++ b/ovk/adapters/ci_secrets/exposure.py @@ -50,8 +50,7 @@ def find_ci_secrets_counterexamples(data: dict[str, Any]) -> list[dict[str, Any] { "summary": str(finding.get("summary") or kind), "failure_mode": FAILURE_MODE, - "workflow_id": ",".join(str(item) for item in finding.get("node_ids") or []) - or "trust-flow", + "workflow_id": ",".join(str(item) for item in finding.get("node_ids") or []) or "trust-flow", "triggers": [], "trust_context": trust_context, "trust_finding_kind": kind, diff --git a/ovk/adapters/ci_secrets/regression.py b/ovk/adapters/ci_secrets/regression.py index bc8a4e1..f2b3041 100644 --- a/ovk/adapters/ci_secrets/regression.py +++ b/ovk/adapters/ci_secrets/regression.py @@ -23,7 +23,10 @@ def render_ci_secrets_regression_test(counterexample: dict[str, Any], index: int def render_ci_secrets_regression_suite(counterexamples: list[dict[str, Any]]) -> str: if not counterexamples: return "# No CI secrets counterexamples were available.\n" - return "\n\n".join( - render_ci_secrets_regression_test(counterexample, index) - for index, counterexample in enumerate(counterexamples) - ) + "\n" + return ( + "\n\n".join( + render_ci_secrets_regression_test(counterexample, index) + for index, counterexample in enumerate(counterexamples) + ) + + "\n" + ) diff --git a/ovk/adapters/deployment/deterministic_adapter.py b/ovk/adapters/deployment/deterministic_adapter.py index 8a73a3d..a1d0d6d 100644 --- a/ovk/adapters/deployment/deterministic_adapter.py +++ b/ovk/adapters/deployment/deterministic_adapter.py @@ -2,11 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.deployment.state_machine import find_skipped_approval_paths from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -24,9 +22,10 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -154,51 +153,22 @@ def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironme native_available=False, ) - def run(self, backend_obligation: BackendObligation, budget: ExecutionBudget) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - if budget.per_backend_wall_time_seconds <= 0: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="timeout", - native_execution=False, - exit_code=1, - raw_result={"status": "unknown", "reason": "budget timeout", "counterexamples": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - data = dict(backend_obligation.payload.get("input") or {}) - if not data.get("states") or not data.get("transitions"): - status = "unknown" - counterexamples = [ - { - "summary": "State machine abstraction is missing states or transitions.", - "failure_mode": "missing_state_machine_abstraction", - } - ] - termination = "invalid_output" - else: - counterexamples = find_skipped_approval_paths(data) - status = "fail" if counterexamples else "pass" - termination = "completed" - raw = RawBackendExecution( + def run( + self, + backend_obligation: BackendObligation, + budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, + ) -> RawBackendExecution: + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination=termination, # type: ignore[arg-type] - native_execution=False, - exit_code=0 if termination == "completed" else 1, - raw_result={"status": status, "counterexamples": counterexamples}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="deployment-deterministic", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, @@ -220,9 +190,7 @@ def normalize( assumptions=["Deterministic deployment state-machine evaluator."], limits=["Does not execute live deployment controllers."], counterexamples=list(raw.raw_result.get("counterexamples") or []), - generated_artifacts=[ - {"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False} - ], + generated_artifacts=[{"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False}], ) def explain(self, result: NormalizedBackendResult) -> HumanExplanation: diff --git a/ovk/adapters/deployment/diff_extract.py b/ovk/adapters/deployment/diff_extract.py index 9f31543..ab07c1c 100644 --- a/ovk/adapters/deployment/diff_extract.py +++ b/ovk/adapters/deployment/diff_extract.py @@ -35,8 +35,7 @@ def _normalize_deployment_payload(states: list[dict]) -> dict: required_states = [str(state["name"]) for state in states if state.get("requires_approval")] production_states = [state_names[-1]] if state_names else ["production"] state_metadata = { - str(state["name"]): {"requires_approval": bool(state.get("requires_approval"))} - for state in states + str(state["name"]): {"requires_approval": bool(state.get("requires_approval"))} for state in states } return { "initial_state": state_names[0] if state_names else "draft", diff --git a/ovk/adapters/deployment/regression.py b/ovk/adapters/deployment/regression.py index 46367a6..32aa8f5 100644 --- a/ovk/adapters/deployment/regression.py +++ b/ovk/adapters/deployment/regression.py @@ -22,7 +22,10 @@ def render_deployment_regression_test(counterexample: dict[str, Any], index: int def render_deployment_regression_suite(counterexamples: list[dict[str, Any]]) -> str: if not counterexamples: return "# No deployment state counterexamples were available.\n" - return "\n\n".join( - render_deployment_regression_test(counterexample, index) - for index, counterexample in enumerate(counterexamples) - ) + "\n" + return ( + "\n\n".join( + render_deployment_regression_test(counterexample, index) + for index, counterexample in enumerate(counterexamples) + ) + + "\n" + ) diff --git a/ovk/adapters/deployment/state_machine.py b/ovk/adapters/deployment/state_machine.py index cf73bf3..fa5a800 100644 --- a/ovk/adapters/deployment/state_machine.py +++ b/ovk/adapters/deployment/state_machine.py @@ -83,8 +83,7 @@ def find_skipped_approval_paths(data: dict[str, Any]) -> list[dict[str, Any]]: counterexamples.append( { "summary": ( - f"Production state {state} is reachable from {previous} " - "without a prior approval gate." + f"Production state {state} is reachable from {previous} without a prior approval gate." ), "failure_mode": FAILURE_MODE, "path": path, diff --git a/ovk/adapters/infra/graph.py b/ovk/adapters/infra/graph.py index 6ef9961..a9cfaad 100644 --- a/ovk/adapters/infra/graph.py +++ b/ovk/adapters/infra/graph.py @@ -40,9 +40,7 @@ def _adjacency(data: dict[str, Any]) -> dict[str, list[str]]: def _entrypoints(nodes: dict[str, dict[str, Any]]) -> list[str]: return [ - node_id - for node_id, node in nodes.items() - if node.get("external") is True or node.get("kind") == "external" + node_id for node_id, node in nodes.items() if node.get("external") is True or node.get("kind") == "external" ] diff --git a/ovk/adapters/infrastructure/deterministic_adapter.py b/ovk/adapters/infrastructure/deterministic_adapter.py index 45adc99..1ace646 100644 --- a/ovk/adapters/infrastructure/deterministic_adapter.py +++ b/ovk/adapters/infrastructure/deterministic_adapter.py @@ -2,12 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.infra.exposure import find_exposure_counterexamples -from ovk.adapters.infra.validation import validate_infra_input from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -25,9 +22,10 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -155,54 +153,22 @@ def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironme native_available=False, ) - def run(self, backend_obligation: BackendObligation, budget: ExecutionBudget) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - if budget.per_backend_wall_time_seconds <= 0: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="timeout", - native_execution=False, - exit_code=1, - raw_result={"status": "unknown", "reason": "budget timeout", "counterexamples": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - data = dict(backend_obligation.payload.get("input") or {}) - issues = validate_infra_input(data) - if issues: - status = "unknown" - counterexamples = [ - { - "summary": issue.message, - "failure_mode": "infrastructure_abstraction_invalid", - "path": issue.path, - } - for issue in issues - ] - termination = "invalid_output" - else: - counterexamples = find_exposure_counterexamples(data) - status = "fail" if counterexamples else "pass" - termination = "completed" - raw = RawBackendExecution( + def run( + self, + backend_obligation: BackendObligation, + budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, + ) -> RawBackendExecution: + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination=termination, # type: ignore[arg-type] - native_execution=False, - exit_code=0 if termination == "completed" else 1, - raw_result={"status": status, "counterexamples": counterexamples}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="infrastructure-deterministic", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, @@ -224,9 +190,7 @@ def normalize( assumptions=["Deterministic infrastructure exposure evaluator."], limits=["Does not claim Terraform plan or Kubernetes API execution."], counterexamples=list(raw.raw_result.get("counterexamples") or []), - generated_artifacts=[ - {"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False} - ], + generated_artifacts=[{"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False}], ) def explain(self, result: NormalizedBackendResult) -> HumanExplanation: diff --git a/ovk/adapters/lane/_base.py b/ovk/adapters/lane/_base.py index ebcc2aa..2d52d14 100644 --- a/ovk/adapters/lane/_base.py +++ b/ovk/adapters/lane/_base.py @@ -186,9 +186,7 @@ def compile( }, expected_guarantee=self.guarantee_type, ) - return provisional.model_copy( - update={"backend_obligation_id": compute_backend_obligation_id(provisional)} - ) + return provisional.model_copy(update={"backend_obligation_id": compute_backend_obligation_id(provisional)}) def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironmentFingerprint: env_payload = { @@ -249,11 +247,6 @@ def run( finished_at = _utc_now_iso() duration_ms = (time.perf_counter() - started) * 1000.0 - # Soft budget recording only; hard isolation arrives with the control plane. - if duration_ms > budget.per_backend_wall_time_seconds * 1000.0 and termination == "completed": - termination = "timeout" - raw_result["status"] = "unknown" - raw_result["timeout"] = True raw = RawBackendExecution( backend=self.backend_id, diff --git a/ovk/adapters/opa/infra_exposure.py b/ovk/adapters/opa/infra_exposure.py index 5c4a2f2..5279b24 100644 --- a/ovk/adapters/opa/infra_exposure.py +++ b/ovk/adapters/opa/infra_exposure.py @@ -9,7 +9,8 @@ from ovk.core.models import VerificationEvidence -INFRA_EXPOSURE_REGO = r''' +INFRA_EXPOSURE_REGO = ( + r""" package ovk.infra_exposure violation[msg] { @@ -25,7 +26,9 @@ resource.public_exposure == true msg := sprintf("sensitive resource publicly exposed: %s", [resource.resource_id]) } -'''.strip() + "\n" +""".strip() + + "\n" +) def _deterministic_infra_opa(data: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]: @@ -33,7 +36,9 @@ def _deterministic_infra_opa(data: dict[str, Any]) -> tuple[str, list[dict[str, if counterexamples: return "fail", counterexamples if not data.get("resources"): - return "unknown", [{"summary": "resources must be a non-empty list", "failure_mode": "infrastructure_abstraction_invalid"}] + return "unknown", [ + {"summary": "resources must be a non-empty list", "failure_mode": "infrastructure_abstraction_invalid"} + ] return "pass", [] diff --git a/ovk/adapters/opa/policy_assets.py b/ovk/adapters/opa/policy_assets.py index 3771bc8..d696d9c 100644 --- a/ovk/adapters/opa/policy_assets.py +++ b/ovk/adapters/opa/policy_assets.py @@ -10,7 +10,8 @@ from pathlib import Path -SELF_PROTECTION_REGO = r''' +SELF_PROTECTION_REGO = ( + r""" package ovk.self_protection after_has_gate(gate) { @@ -38,7 +39,9 @@ input.after.workflow_permissions.actions == "write" msg := "workflow actions permission escalated to write" } -'''.strip() + "\n" +""".strip() + + "\n" +) def write_self_protection_rego(path: Path) -> None: diff --git a/ovk/adapters/opa/self_protection.py b/ovk/adapters/opa/self_protection.py index 3f00c38..0239605 100644 --- a/ovk/adapters/opa/self_protection.py +++ b/ovk/adapters/opa/self_protection.py @@ -133,7 +133,9 @@ def find_self_protection_violations(data: dict[str, Any]) -> list[SelfProtection permissions_after = _phase(data, "after").get("workflow_permissions", {}) permissions_before = _phase(data, "before").get("workflow_permissions", {}) if isinstance(permissions_after, dict): - before_actions = str(permissions_before.get("actions", "read")) if isinstance(permissions_before, dict) else "read" + before_actions = ( + str(permissions_before.get("actions", "read")) if isinstance(permissions_before, dict) else "read" + ) after_actions = str(permissions_after.get("actions", "read")) if before_actions != "write" and after_actions == "write": violations.append( diff --git a/ovk/adapters/self_protection/deterministic_adapter.py b/ovk/adapters/self_protection/deterministic_adapter.py index cf75b0b..b5359d1 100644 --- a/ovk/adapters/self_protection/deterministic_adapter.py +++ b/ovk/adapters/self_protection/deterministic_adapter.py @@ -2,14 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.opa.self_protection import ( - find_self_protection_unknowns, - find_self_protection_violations, -) from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -27,9 +22,10 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -107,11 +103,7 @@ def can_handle( estimated_memory_mb=64, reasons=["excluded by execution budget"], ) - trusted_meta = all( - item.trusted - for item in obligation.materials - if item.kind == "branch_protection" - ) + trusted_meta = all(item.trusted for item in obligation.materials if item.kind == "branch_protection") materials_ok = bool(obligation.materials) coverage_ok = obligation.coverage.status in {"complete", "partial", "unknown"} score = 0.8 if trusted_meta else 0.55 @@ -160,49 +152,22 @@ def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironme native_available=False, ) - def run(self, backend_obligation: BackendObligation, budget: ExecutionBudget) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - if budget.per_backend_wall_time_seconds <= 0: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="timeout", - native_execution=False, - exit_code=1, - raw_result={"status": "unknown", "reason": "budget timeout", "counterexamples": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - data = dict(backend_obligation.payload.get("input") or {}) - violations = find_self_protection_violations(data) - unknowns = find_self_protection_unknowns(data) - if violations: - status = "fail" - counterexamples = [item.as_counterexample() for item in violations] - elif unknowns: - status = "unknown" - counterexamples = [item.as_counterexample() for item in unknowns] - else: - status = "pass" - counterexamples = [] - raw = RawBackendExecution( + def run( + self, + backend_obligation: BackendObligation, + budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, + ) -> RawBackendExecution: + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination="completed", - native_execution=False, - exit_code=0, - raw_result={"status": status, "counterexamples": counterexamples}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="self-protection-deterministic", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, @@ -224,9 +189,7 @@ def normalize( assumptions=["Deterministic self-protection evaluator."], limits=["OPA native execution is a separate backend."], counterexamples=list(raw.raw_result.get("counterexamples") or []), - generated_artifacts=[ - {"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False} - ], + generated_artifacts=[{"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False}], ) def explain(self, result: NormalizedBackendResult) -> HumanExplanation: diff --git a/ovk/adapters/self_protection/opa_adapter.py b/ovk/adapters/self_protection/opa_adapter.py index 8317633..3fe8895 100644 --- a/ovk/adapters/self_protection/opa_adapter.py +++ b/ovk/adapters/self_protection/opa_adapter.py @@ -30,6 +30,7 @@ compute_payload_digest, compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus @@ -173,7 +174,8 @@ def run( self, backend_obligation: BackendObligation, budget: ExecutionBudget, - worker=None, + *, + worker: BackendWorker | None = None, ) -> RawBackendExecution: started = time.perf_counter() started_at = _utc_now_iso() @@ -192,6 +194,24 @@ def run( ) return raw.model_copy(update=compute_raw_execution_digests(raw)) + if worker is None: + raw = RawBackendExecution( + backend=self.backend_id, + backend_obligation_id=backend_obligation.backend_obligation_id, + termination="tool_error", + native_execution=False, + exit_code=1, + raw_result={ + "status": "error", + "reason": "authoritative OPA adapter requires BackendWorker", + "counterexamples": [], + }, + started_at=started_at, + finished_at=_utc_now_iso(), + duration_ms=(time.perf_counter() - started) * 1000.0, + ) + return raw.model_copy(update=compute_raw_execution_digests(raw)) + if not opa_available(): raw = RawBackendExecution( backend=self.backend_id, @@ -222,9 +242,7 @@ def run( status = str(result.get("status", "unknown")) violations = result.get("violations") or [] counterexamples = [ - {"summary": str(item), "failure_mode": "opa_policy_violation"} - if not isinstance(item, dict) - else item + {"summary": str(item), "failure_mode": "opa_policy_violation"} if not isinstance(item, dict) else item for item in violations ] termination = "completed" @@ -293,7 +311,9 @@ def explain(self, result: NormalizedBackendResult) -> HumanExplanation: failure_mode=str(result.counterexamples[0].get("failure_mode", "opa_policy_violation")), ) if result.status == VerificationStatus.PASS: - return HumanExplanation(summary="OPA reported no self-protection violations.", repair_hint="No repair required.") + return HumanExplanation( + summary="OPA reported no self-protection violations.", repair_hint="No repair required." + ) return HumanExplanation( summary=f"OPA-native backend returned {result.status.value}.", repair_hint="Install opa or select an explicitly accepted fallback backend via policy.", diff --git a/ovk/adapters/z3/obligation.py b/ovk/adapters/z3/obligation.py index 13d6e1a..043f13c 100644 --- a/ovk/adapters/z3/obligation.py +++ b/ovk/adapters/z3/obligation.py @@ -82,10 +82,7 @@ def obligation_to_dict(obligation: AuthorizationObligation) -> dict[str, Any]: "path": route.path, "admin_only_before": route.admin_only_before, "admin_only_after": route.admin_only_after, - "reachable_after": [ - {"role": witness.role, "via": witness.via} - for witness in route.reachable_after - ], + "reachable_after": [{"role": witness.role, "via": witness.via} for witness in route.reachable_after], } for route in obligation.routes ], diff --git a/ovk/adapters/z3/regression.py b/ovk/adapters/z3/regression.py index 28cf5de..24deda2 100644 --- a/ovk/adapters/z3/regression.py +++ b/ovk/adapters/z3/regression.py @@ -26,5 +26,9 @@ def render_authorization_regression_suite(counterexamples: list[dict[str, Any]]) tests = [] for index, counterexample in enumerate(counterexamples): test_body = render_authorization_regression_test(counterexample) - tests.append(test_body.replace("test_non_admin_cannot_reach_admin_route", f"test_non_admin_cannot_reach_admin_route_{index}")) + tests.append( + test_body.replace( + "test_non_admin_cannot_reach_admin_route", f"test_non_admin_cannot_reach_admin_route_{index}" + ) + ) return "\n\n".join(tests) + "\n" diff --git a/ovk/adapters/z3/smt_plan.py b/ovk/adapters/z3/smt_plan.py index b1aeb01..98a15b2 100644 --- a/ovk/adapters/z3/smt_plan.py +++ b/ovk/adapters/z3/smt_plan.py @@ -56,8 +56,5 @@ def smt_plan_to_dict(plan: SmtPlan) -> dict: return { "obligation_id": plan.obligation_id, "query_polarity": plan.query_polarity, - "clauses": [ - {"name": clause.name, "expression": clause.expression} - for clause in plan.clauses - ], + "clauses": [{"name": clause.name, "expression": clause.expression} for clause in plan.clauses], } diff --git a/ovk/cli.py b/ovk/cli.py index 4c91747..8122f1b 100644 --- a/ovk/cli.py +++ b/ovk/cli.py @@ -298,7 +298,9 @@ def release_bundle( base_sha: Optional[str] = typer.Option(None), ) -> None: """Write a complete verifiable release bundle for a lane.""" - evidence = _evaluate_lane(lane, input_json, input_format=input_format, policy=policy, repo=repo, head_sha=head_sha, base_sha=base_sha) + evidence = _evaluate_lane( + lane, input_json, input_format=input_format, policy=policy, repo=repo, head_sha=head_sha, base_sha=base_sha + ) bundle = make_bundle([evidence]) write_release_bundle(bundle, ReleaseBundlePaths(root=output_dir)) failures = verify_release_bundle(output_dir) @@ -324,7 +326,9 @@ def _evaluate_lane( return evaluate_validated_authorization_path(data, repo=repo, head_sha=head_sha, base_sha=base_sha) if lane == "infrastructure": normalized = normalize_infra_input(data, input_format) - return evaluate_infra_exposure(normalized, repo=repo, head_sha=head_sha, base_sha=base_sha, policy=load_policy(policy)) + return evaluate_infra_exposure( + normalized, repo=repo, head_sha=head_sha, base_sha=base_sha, policy=load_policy(policy) + ) if lane == "self_protection": return evaluate_self_protection(data, repo=repo, head_sha=head_sha, base_sha=base_sha) if lane == "ci_secrets": @@ -409,8 +413,12 @@ def auth_obligation( evidence_output: Path = typer.Option(Path("ovk-auth-evidence.json"), help="Evidence bundle output path."), markdown_output: Path = typer.Option(Path("ovk-auth-comment.md"), help="Markdown output path."), attestation_output: Path = typer.Option(Path("ovk-auth-attestation.json"), help="Attestation output path."), - manifest_output: Path = typer.Option(Path("ovk-auth-artifact-manifest.json"), help="Artifact manifest output path."), - quality_output: Path = typer.Option(Path("ovk-auth-evidence-quality.json"), help="Evidence quality report output path."), + manifest_output: Path = typer.Option( + Path("ovk-auth-artifact-manifest.json"), help="Artifact manifest output path." + ), + quality_output: Path = typer.Option( + Path("ovk-auth-evidence-quality.json"), help="Evidence quality report output path." + ), advisory: bool = typer.Option(False, help="Write outputs and exit 0."), ) -> None: """Run the validated authorization obligation path.""" @@ -439,8 +447,12 @@ def infra_exposure( evidence_output: Path = typer.Option(Path("ovk-infra-evidence.json"), help="Evidence bundle output path."), markdown_output: Path = typer.Option(Path("ovk-infra-comment.md"), help="Markdown output path."), attestation_output: Path = typer.Option(Path("ovk-infra-attestation.json"), help="Attestation output path."), - manifest_output: Path = typer.Option(Path("ovk-infra-artifact-manifest.json"), help="Artifact manifest output path."), - quality_output: Path = typer.Option(Path("ovk-infra-evidence-quality.json"), help="Evidence quality report output path."), + manifest_output: Path = typer.Option( + Path("ovk-infra-artifact-manifest.json"), help="Artifact manifest output path." + ), + quality_output: Path = typer.Option( + Path("ovk-infra-evidence-quality.json"), help="Evidence quality report output path." + ), advisory: bool = typer.Option(False, help="Write outputs and exit 0."), ) -> None: """Run the infrastructure exposure path.""" @@ -473,8 +485,12 @@ def ci_secrets( evidence_output: Path = typer.Option(Path("ovk-ci-secrets-evidence.json"), help="Evidence bundle output path."), markdown_output: Path = typer.Option(Path("ovk-ci-secrets-comment.md"), help="Markdown output path."), attestation_output: Path = typer.Option(Path("ovk-ci-secrets-attestation.json"), help="Attestation output path."), - manifest_output: Path = typer.Option(Path("ovk-ci-secrets-artifact-manifest.json"), help="Artifact manifest output path."), - quality_output: Path = typer.Option(Path("ovk-ci-secrets-evidence-quality.json"), help="Evidence quality report output path."), + manifest_output: Path = typer.Option( + Path("ovk-ci-secrets-artifact-manifest.json"), help="Artifact manifest output path." + ), + quality_output: Path = typer.Option( + Path("ovk-ci-secrets-evidence-quality.json"), help="Evidence quality report output path." + ), advisory: bool = typer.Option(False, help="Write outputs and exit 0."), ) -> None: """Run the CI secrets exposure path.""" @@ -500,12 +516,18 @@ def deployment_state( evidence_output: Path = typer.Option(Path("ovk-deployment-evidence.json"), help="Evidence bundle output path."), markdown_output: Path = typer.Option(Path("ovk-deployment-comment.md"), help="Markdown output path."), attestation_output: Path = typer.Option(Path("ovk-deployment-attestation.json"), help="Attestation output path."), - manifest_output: Path = typer.Option(Path("ovk-deployment-artifact-manifest.json"), help="Artifact manifest output path."), - quality_output: Path = typer.Option(Path("ovk-deployment-evidence-quality.json"), help="Evidence quality report output path."), + manifest_output: Path = typer.Option( + Path("ovk-deployment-artifact-manifest.json"), help="Artifact manifest output path." + ), + quality_output: Path = typer.Option( + Path("ovk-deployment-evidence-quality.json"), help="Evidence quality report output path." + ), advisory: bool = typer.Option(False, help="Write outputs and exit 0."), ) -> None: """Run the deployment approval state machine path.""" - evidence = evaluate_approval_state_machine(read_json_file(input_json), repo=repo, head_sha=head_sha, base_sha=base_sha) + evidence = evaluate_approval_state_machine( + read_json_file(input_json), repo=repo, head_sha=head_sha, base_sha=base_sha + ) _finish_lane( make_bundle([evidence]), label="deployment", @@ -725,7 +747,9 @@ def run_cmd( @app.command("generate-test") def generate_test( evidence_bundle: Path = typer.Option(..., "--evidence", help="Evidence bundle JSON."), - output_dir: Path = typer.Option(Path(".verification/generated_tests"), help="Regression artifact output directory."), + output_dir: Path = typer.Option( + Path(".verification/generated_tests"), help="Regression artifact output directory." + ), ) -> None: """Generate regression artifacts from bundle counterexamples.""" bundle = EvidenceBundle.model_validate(read_json_file(evidence_bundle)) @@ -740,11 +764,7 @@ def repair_suggest( ) -> None: """Emit machine-readable repair hints from bundle counterexamples.""" bundle = EvidenceBundle.model_validate(read_json_file(evidence_bundle)) - counterexamples = [ - counterexample - for evidence in bundle.evidence - for counterexample in evidence.counterexamples - ] + counterexamples = [counterexample for evidence in bundle.evidence for counterexample in evidence.counterexamples] if not counterexamples: typer.echo(json.dumps({"repair_hints": [], "blocked": bundle.decision.get("merge_recommendation") == "block"})) return diff --git a/ovk/compilers/authorization/__init__.py b/ovk/compilers/authorization/__init__.py index 908b30d..86f243b 100644 --- a/ovk/compilers/authorization/__init__.py +++ b/ovk/compilers/authorization/__init__.py @@ -5,6 +5,7 @@ from ovk.compilers.authorization.coverage import CoveragePolicy, assess_coverage, strict_allow_permitted from ovk.compilers.authorization.express import ExpressAuthorizationCompiler from ovk.compilers.authorization.fastapi import FastApiAuthorizationCompiler +from ovk.compilers.authorization.fastapi_ast import FastApiAstAuthorizationCompiler from ovk.compilers.authorization.ir import AuthorizationIR from ovk.compilers.authorization.material_loader import ( AuthMaterials, @@ -17,6 +18,7 @@ "AuthorizationIR", "CoveragePolicy", "ExpressAuthorizationCompiler", + "FastApiAstAuthorizationCompiler", "FastApiAuthorizationCompiler", "assess_coverage", "load_materials_from_dirs", diff --git a/ovk/compilers/authorization/corpus.py b/ovk/compilers/authorization/corpus.py index cf70b85..973191b 100644 --- a/ovk/compilers/authorization/corpus.py +++ b/ovk/compilers/authorization/corpus.py @@ -187,11 +187,7 @@ def build_corpus(*, meet_targets: bool = True) -> list[CorpusCase]: def classify_case(case: CorpusCase) -> Category: - compiler = ( - FastApiAuthorizationCompiler() - if case.framework == "fastapi" - else ExpressAuthorizationCompiler() - ) + compiler = FastApiAuthorizationCompiler() if case.framework == "fastapi" else ExpressAuthorizationCompiler() materials = materials_from_pair( path=case.path, base_source=case.base if case.base else None, diff --git a/ovk/compilers/authorization/express.py b/ovk/compilers/authorization/express.py index 417c38e..07c6c93 100644 --- a/ovk/compilers/authorization/express.py +++ b/ovk/compilers/authorization/express.py @@ -76,7 +76,9 @@ def _index(self, files: dict[str, str]) -> dict: for path, source in sorted(files.items()): for match in _IMPORT.finditer(source): - name = match.group("named") or match.group("default") or match.group("imnamed") or match.group("imdefault") + name = ( + match.group("named") or match.group("default") or match.group("imnamed") or match.group("imdefault") + ) mod = match.group("mod") or match.group("immod") if not name: continue diff --git a/ovk/compilers/authorization/fastapi_ast.py b/ovk/compilers/authorization/fastapi_ast.py new file mode 100644 index 0000000..64ef41b --- /dev/null +++ b/ovk/compilers/authorization/fastapi_ast.py @@ -0,0 +1,298 @@ +"""AST-based FastAPI authorization compiler (profile ``authorization.fastapi.ast_v1``). + +Prefer this over the regex FastAPI compiler when source profiles are enabled. +Dynamic path construction, runtime-computed dependencies, and unrecognized +auth helpers are marked ``dynamic`` / ``unsupported``. +""" + +from __future__ import annotations + +import ast +from typing import Any + +from ovk.compilers.authorization.base import looks_admin_protected, normalize_path +from ovk.compilers.authorization.ir import ( + AuthCheck, + AuthDependency, + AuthMount, + AuthRoute, + AuthorizationIR, + SourceSpan, +) +from ovk.compilers.authorization.material_loader import AuthMaterials + +_HTTP_METHODS = frozenset({"get", "post", "put", "patch", "delete", "options", "head"}) +_SOURCE_PROFILE_ID = "authorization.fastapi.ast_v1" + + +def _const_str(node: ast.AST | None) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return None + + +def _name_of(node: ast.AST | None) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return None + + +def _call_name(node: ast.Call) -> str | None: + return _name_of(node.func) + + +def _kw_str(call: ast.Call, name: str) -> str | None: + for keyword in call.keywords: + if keyword.arg == name: + return _const_str(keyword.value) + return None + + +def _depends_names(node: ast.AST | None) -> list[str]: + found: list[str] = [] + if node is None: + return found + for child in ast.walk(node): + if isinstance(child, ast.Call) and _call_name(child) == "Depends" and child.args: + dep = _name_of(child.args[0]) + if dep: + found.append(dep) + return found + + +def _span(path: str, node: ast.AST) -> SourceSpan: + return SourceSpan( + path=path, + start_line=getattr(node, "lineno", None), + end_line=getattr(node, "end_lineno", getattr(node, "lineno", None)), + ) + + +class FastApiAstAuthorizationCompiler: + """Compile FastAPI sources via the Python AST (not regex).""" + + framework = "fastapi" + source_profile_id = _SOURCE_PROFILE_ID + + def compile(self, materials: AuthMaterials) -> AuthorizationIR: + base_index = self._index(materials.base_files) + head_index = self._index(materials.head_files) + routes = self._merge_routes(base_index, head_index) + mounts = sorted( + {**base_index["mounts"], **head_index["mounts"]}.values(), + key=lambda item: item.mount_id, + ) + dependencies = sorted( + {**base_index["dependencies"], **head_index["dependencies"]}.values(), + key=lambda item: item.name, + ) + unsupported = sorted(set(base_index["unsupported"] + head_index["unsupported"])) + warnings: list[str] = [f"compiled_with_source_profile:{_SOURCE_PROFILE_ID}"] + if not materials.has_base(): + warnings.append("base materials missing") + if not materials.has_head(): + warnings.append("head materials missing") + return AuthorizationIR( + framework="fastapi", + subject_repo=materials.repo, + base_revision=materials.base_revision, + head_revision=materials.head_revision, + routes=sorted(routes, key=lambda item: (item.path, ",".join(item.methods), item.route_id)), + mounts=list(mounts), + dependencies=list(dependencies), + unsupported_constructs=unsupported, + warnings=warnings, + materials=materials.paths, + ) + + def _index(self, files: dict[str, str]) -> dict[str, Any]: + routers: dict[str, str] = {} + mounts: dict[str, AuthMount] = {} + dependencies: dict[str, AuthDependency] = {} + route_map: dict[tuple[str, str], dict[str, Any]] = {} + unsupported: list[str] = [] + + for path, source in sorted(files.items()): + try: + tree = ast.parse(source, filename=path) + except SyntaxError as exc: + unsupported.append(f"{path}:syntax_error:{exc.msg}") + continue + + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + roles = ( + ["admin"] if looks_admin_protected(ast.get_source_segment(source, node) or node.name) else [] + ) + dependencies[node.name] = AuthDependency( + name=node.name, + kind="dependency", + role_checks=roles, + support="supported", + ) + + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Call): + if _call_name(node.value) == "APIRouter": + for target in node.targets: + name = _name_of(target) + if not name: + continue + prefix = _kw_str(node.value, "prefix") or "" + routers[name] = prefix + deps = _depends_names(node.value) + if deps: + mounts[f"{path}:{name}"] = AuthMount( + mount_id=f"{path}:{name}", + prefix=prefix, + middleware=deps, + included_router=name, + ) + + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + if node.func.attr == "include_router" and node.args: + router = _name_of(node.args[0]) + if not router: + continue + prefix = _kw_str(node, "prefix") or routers.get(router, "") + mounts[f"{path}:include:{router}"] = AuthMount( + mount_id=f"{path}:include:{router}", + prefix=prefix, + included_router=router, + ) + routers.setdefault(router, prefix) + + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for decorator in node.decorator_list: + route = self._route_from_decorator( + decorator, + handler=node, + path=path, + source=source, + routers=routers, + dependencies=dependencies, + unsupported=unsupported, + ) + if route is None: + continue + key = (route["path"], route["method"]) + route_map[key] = route + + return { + "routes": route_map, + "mounts": mounts, + "dependencies": dependencies, + "unsupported": unsupported, + } + + def _route_from_decorator( + self, + decorator: ast.AST, + *, + handler: ast.AST, + path: str, + source: str, + routers: dict[str, str], + dependencies: dict[str, AuthDependency], + unsupported: list[str], + ) -> dict[str, Any] | None: + if not isinstance(decorator, ast.Call) or not isinstance(decorator.func, ast.Attribute): + return None + method = decorator.func.attr.lower() + if method not in _HTTP_METHODS: + return None + app = _name_of(decorator.func.value) or "app" + if not decorator.args: + unsupported.append(f"{path}:dynamic_route_path") + return None + route_path = _const_str(decorator.args[0]) + if route_path is None: + unsupported.append(f"{path}:dynamic_route_path") + return None + + prefix = routers.get(app, "") + full_path = normalize_path(prefix, route_path) + deps = _depends_names(decorator) + # Also inspect handler signature defaults for Depends(...). + if isinstance(handler, (ast.FunctionDef, ast.AsyncFunctionDef)): + for arg in list(handler.args.args) + list(handler.args.kwonlyargs): + deps.extend(_depends_names(arg.annotation)) + for default in list(handler.args.defaults) + list(handler.args.kw_defaults): + deps.extend(_depends_names(default)) + # Deduplicate while preserving order. + seen: set[str] = set() + ordered_deps: list[str] = [] + for dep in deps: + if dep not in seen: + seen.add(dep) + ordered_deps.append(dep) + + checks: list[AuthCheck] = [] + admin = False + for dep in ordered_deps: + dep_meta = dependencies.get(dep) + roles = list(dep_meta.role_checks) if dep_meta else [] + if looks_admin_protected(dep) or roles: + admin = True + roles = roles or ["admin"] + checks.append( + AuthCheck( + kind="dependency", + expression=f"Depends({dep})", + roles=roles, + span=_span(path, decorator), + ) + ) + + handler_name = getattr(handler, "name", None) + if handler_name: + handler_meta = dependencies.get(handler_name) + if handler_meta and handler_meta.role_checks: + admin = True + body_text = ast.get_source_segment(source, handler) or "" + if looks_admin_protected(body_text): + admin = True + + return { + "path": full_path, + "method": method.upper(), + "handler": handler_name, + "prefixes": [prefix] if prefix else [], + "checks": checks, + "dependencies": ordered_deps, + "admin_only": admin, + "support": "supported", + "unsupported": [], + "span": _span(path, decorator), + "source_path": path, + } + + def _merge_routes(self, base_index: dict[str, Any], head_index: dict[str, Any]) -> list[AuthRoute]: + keys = sorted(set(base_index["routes"]) | set(head_index["routes"])) + routes: list[AuthRoute] = [] + for index, key in enumerate(keys): + before = base_index["routes"].get(key) + after = head_index["routes"].get(key) + path = (after or before)["path"] + method = (after or before)["method"] + routes.append( + AuthRoute( + route_id=f"fastapi-ast:{path}:{method}:{index}", + methods=[method], + path=path, + handler=(after or before).get("handler"), + router_prefixes=(after or before).get("prefixes") or [], + checks_before=list((before or {}).get("checks") or []), + checks_after=list((after or {}).get("checks") or []), + dependencies_before=list((before or {}).get("dependencies") or []), + dependencies_after=list((after or {}).get("dependencies") or []), + admin_only_before=bool((before or {}).get("admin_only")), + admin_only_after=bool((after or {}).get("admin_only")), + support=(after or before).get("support", "supported"), + unsupported_constructs=list((after or before).get("unsupported") or []), + span=(after or before).get("span"), + ) + ) + return routes diff --git a/ovk/compilers/cbmc/traceability.py b/ovk/compilers/cbmc/traceability.py index f7ffe01..c8ef263 100644 --- a/ovk/compilers/cbmc/traceability.py +++ b/ovk/compilers/cbmc/traceability.py @@ -36,7 +36,5 @@ def validate_project_traceability(project: CbmcProject) -> list[str]: if record.get(field) in (None, [], ""): failures.append(f"{harness.harness_id}: missing {field}") if project.guarantee_type == "bounded_project_model_check" and not harness.includes_project_code: - failures.append( - f"{harness.harness_id}: bounded_project_model_check requires includes_project_code" - ) + failures.append(f"{harness.harness_id}: bounded_project_model_check requires includes_project_code") return failures diff --git a/ovk/compilers/deployment/ir.py b/ovk/compilers/deployment/ir.py index 88aa37d..9347bb3 100644 --- a/ovk/compilers/deployment/ir.py +++ b/ovk/compilers/deployment/ir.py @@ -37,10 +37,7 @@ def to_lane_input(self) -> dict[str, Any]: return { "initial_state": self.initial_state, "states": [item.name for item in self.states], - "transitions": [ - {"from": item.source, "to": item.target, "label": item.label} - for item in self.transitions - ], + "transitions": [{"from": item.source, "to": item.target, "label": item.label} for item in self.transitions], "required_states": list(self.required_states), "production_states": list(self.production_states), "warnings": list(self.warnings), diff --git a/ovk/compilers/github_actions/composite_actions.py b/ovk/compilers/github_actions/composite_actions.py index 83ea06b..de3e447 100644 --- a/ovk/compilers/github_actions/composite_actions.py +++ b/ovk/compilers/github_actions/composite_actions.py @@ -31,9 +31,7 @@ def expand_composite_action( nodes: list[TrustNode] = [ TrustNode(node_id=f"action:{action_path.as_posix()}", kind="composite_action", trust="unknown") ] - edges: list[TrustEdge] = [ - TrustEdge(source=step_node_id, target=nodes[0].node_id, kind="uses_composite") - ] + edges: list[TrustEdge] = [TrustEdge(source=step_node_id, target=nodes[0].node_id, kind="uses_composite")] secrets: list[SecretUse] = [] for index, step in enumerate(runs.get("steps") or []): if not isinstance(step, dict): diff --git a/ovk/compilers/github_actions/permissions.py b/ovk/compilers/github_actions/permissions.py index 9b7adab..c6ed039 100644 --- a/ovk/compilers/github_actions/permissions.py +++ b/ovk/compilers/github_actions/permissions.py @@ -33,6 +33,5 @@ def _from_block(block: Any, *, job_id: str | None) -> list[PermissionGrant]: if not isinstance(block, dict): return [] return [ - PermissionGrant(scope=str(scope), level=str(level), job_id=job_id) - for scope, level in sorted(block.items()) + PermissionGrant(scope=str(scope), level=str(level), job_id=job_id) for scope, level in sorted(block.items()) ] diff --git a/ovk/compilers/github_actions/reusable_workflows.py b/ovk/compilers/github_actions/reusable_workflows.py index 2f0e9d0..1c156de 100644 --- a/ovk/compilers/github_actions/reusable_workflows.py +++ b/ovk/compilers/github_actions/reusable_workflows.py @@ -105,9 +105,7 @@ def resolve_local_reusable( child = load_workflow_file(local_path) child["_ovk_path"] = str(local_path.resolve()) loaded.append(child) - nested, nested_findings = resolve_local_reusable( - child, repo_root=repo_root, visiting=set(visiting) - ) + nested, nested_findings = resolve_local_reusable(child, repo_root=repo_root, visiting=set(visiting)) loaded.extend(nested) findings.extend(nested_findings) if path: diff --git a/ovk/compilers/github_actions/secrets.py b/ovk/compilers/github_actions/secrets.py index daf47c1..c55a78c 100644 --- a/ovk/compilers/github_actions/secrets.py +++ b/ovk/compilers/github_actions/secrets.py @@ -49,10 +49,7 @@ def _scan(value: Any, *, job_id: str | None, step_id: str | None) -> list[Secret if value is None: return [] if isinstance(value, str): - return [ - SecretUse(name=name, job_id=job_id, step_id=step_id, expression=value) - for name in secret_names(value) - ] + return [SecretUse(name=name, job_id=job_id, step_id=step_id, expression=value) for name in secret_names(value)] if isinstance(value, dict): found: list[SecretUse] = [] for nested in value.values(): diff --git a/ovk/compilers/github_actions/trust_flow.py b/ovk/compilers/github_actions/trust_flow.py index d84d698..bb9a854 100644 --- a/ovk/compilers/github_actions/trust_flow.py +++ b/ovk/compilers/github_actions/trust_flow.py @@ -51,9 +51,7 @@ def compile_workflow_trust( permissions = extract_permissions(workflow) secrets = extract_secrets(workflow) - write_token = has_write_token(permissions) or any( - references_github_token(item.expression) for item in secrets - ) + write_token = has_write_token(permissions) or any(references_github_token(item.expression) for item in secrets) jobs = workflow.get("jobs") if isinstance(workflow.get("jobs"), dict) else {} for job_id, job in sorted(jobs.items()): @@ -102,8 +100,10 @@ def compile_workflow_trust( step_node_id = f"job:{job_id}:step:{step_id}" run = str(step.get("run") or "") uses = str(step.get("uses") or "") - untrusted_code = untrusted_trigger or contains_untrusted_context(run) or contains_untrusted_context( - str(step.get("with") or "") + untrusted_code = ( + untrusted_trigger + or contains_untrusted_context(run) + or contains_untrusted_context(str(step.get("with") or "")) ) step_node = TrustNode( node_id=step_node_id, @@ -174,9 +174,7 @@ def compile_workflow_trust( # resolve_local_reusable already walked the graph with cycle prevention. for child in reusable: child_path = str(child.get("_ovk_path") or "reusable.yml") - nodes.append( - TrustNode(node_id=f"workflow:{child_path}", kind="reusable_workflow_doc", trust="unknown") - ) + nodes.append(TrustNode(node_id=f"workflow:{child_path}", kind="reusable_workflow_doc", trust="unknown")) child_secrets = extract_secrets(child) child_permissions = extract_permissions(child) secrets.extend(child_secrets) diff --git a/ovk/compilers/infrastructure/exposure_graph.py b/ovk/compilers/infrastructure/exposure_graph.py index b34d62b..aedfe7c 100644 --- a/ovk/compilers/infrastructure/exposure_graph.py +++ b/ovk/compilers/infrastructure/exposure_graph.py @@ -95,9 +95,7 @@ def apply_concrete_exposure(resources: list[InfraResourceIR], paths: list[Exposu updated: list[InfraResourceIR] = [] for resource in resources: path_labels = [ - " -> ".join(path.nodes) - for path in paths - if path.is_concrete and path.nodes[-1] == resource.resource_id + " -> ".join(path.nodes) for path in paths if path.is_concrete and path.nodes[-1] == resource.resource_id ] updated.append( resource.model_copy( diff --git a/ovk/compilers/infrastructure/kubernetes.py b/ovk/compilers/infrastructure/kubernetes.py index 2b6a68a..3306710 100644 --- a/ovk/compilers/infrastructure/kubernetes.py +++ b/ovk/compilers/infrastructure/kubernetes.py @@ -2,6 +2,10 @@ Consumes Service, Ingress, Gateway API, NetworkPolicy, RBAC, ServiceAccount, Secret refs, pod security, and admission metadata when present as objects. + +Profile ``infrastructure.kubernetes.controller_reachability_v1`` adds +controller-aware edges from public Services to matching Deployment/StatefulSet/ +DaemonSet workloads via label selectors. """ from __future__ import annotations @@ -17,6 +21,9 @@ from ovk.compilers.infrastructure.reachability import evaluate_eligibility from ovk.compilers.infrastructure.sensitivity import normalize_sensitivity +_SOURCE_PROFILE_ID = "infrastructure.kubernetes.controller_reachability_v1" +_CONTROLLER_KINDS = frozenset({"Deployment", "StatefulSet", "DaemonSet", "ReplicaSet"}) + def _meta(obj: dict[str, Any]) -> dict[str, Any]: meta = obj.get("metadata") @@ -121,8 +128,7 @@ def compile_kubernetes_objects(objects: list[dict[str, Any]] | dict[str, Any]) - attributes["pod_security_labels"] = { key: value for key, value in _meta(obj).get("labels", {}).items() - if isinstance(_meta(obj).get("labels"), dict) - and str(key).startswith("pod-security.kubernetes.io/") + if isinstance(_meta(obj).get("labels"), dict) and str(key).startswith("pod-security.kubernetes.io/") } else: unsupported.append(f"{resource_id}:unsupported_kind:{kind}") @@ -140,6 +146,11 @@ def compile_kubernetes_objects(objects: list[dict[str, Any]] | dict[str, Any]) - ) ) + controller_edges = _controller_reachability_edges(objects_list, resources) + if controller_edges: + edges.extend(controller_edges) + warnings.append(f"compiled_with_source_profile:{_SOURCE_PROFILE_ID}") + all_edges = build_edges(resources, edges) paths = concrete_public_paths(resources, all_edges) resources = apply_concrete_exposure(resources, paths) @@ -152,3 +163,77 @@ def compile_kubernetes_objects(objects: list[dict[str, Any]] | dict[str, Any]) - warnings=warnings, ) return evaluate_eligibility(ir) + + +def _labels(obj: dict[str, Any]) -> dict[str, str]: + labels = _meta(obj).get("labels") + if not isinstance(labels, dict): + return {} + return {str(key): str(value) for key, value in labels.items()} + + +def _selector_match(selector: dict[str, Any] | None, labels: dict[str, str]) -> bool: + if not isinstance(selector, dict) or not selector: + return False + match_labels = selector.get("matchLabels") + if isinstance(match_labels, dict): + return all(labels.get(str(key)) == str(value) for key, value in match_labels.items()) + # Service selectors are flat maps. + return all(labels.get(str(key)) == str(value) for key, value in selector.items()) + + +def _controller_reachability_edges( + objects: list[Any], + resources: list[InfraResourceIR], +) -> list[ExposureEdge]: + """Link public Services to controllers whose pod template labels match.""" + resource_ids = {item.resource_id for item in resources} + services: list[tuple[str, dict[str, Any]]] = [] + controllers: list[tuple[str, dict[str, Any]]] = [] + for index, obj in enumerate(objects): + if not isinstance(obj, dict): + continue + kind = str(obj.get("kind") or "") + resource_id = _name(obj, index) + spec = obj.get("spec") if isinstance(obj.get("spec"), dict) else {} + if kind == "Service": + services.append((resource_id, spec if isinstance(spec, dict) else {})) + elif kind in _CONTROLLER_KINDS: + controllers.append((resource_id, obj)) + + edges: list[ExposureEdge] = [] + for service_id, service_spec in services: + if service_id not in resource_ids: + continue + selector = service_spec.get("selector") + if not isinstance(selector, dict): + continue + for controller_id, controller in controllers: + template = controller.get("spec", {}).get("template") if isinstance(controller.get("spec"), dict) else None + labels = _labels(template) if isinstance(template, dict) else {} + if not _selector_match(selector, labels): + continue + # Ensure controller appears as a resource so paths can terminate. + if controller_id not in resource_ids: + resources.append( + InfraResourceIR( + resource_id=controller_id, + resource_type=str(controller.get("kind") or "Controller"), + kind="pod_security", + sensitivity=_sensitivity(controller), + attributes={ + "source_profile": _SOURCE_PROFILE_ID, + "controller_kind": controller.get("kind"), + }, + ) + ) + resource_ids.add(controller_id) + edges.append( + ExposureEdge( + source=service_id, + target=controller_id, + kind="service_selector", + evidence=f"compiled_with_source_profile:{_SOURCE_PROFILE_ID}", + ) + ) + return edges diff --git a/ovk/compilers/infrastructure/reachability.py b/ovk/compilers/infrastructure/reachability.py index e9b32fd..5120c60 100644 --- a/ovk/compilers/infrastructure/reachability.py +++ b/ovk/compilers/infrastructure/reachability.py @@ -27,7 +27,11 @@ def evaluate_eligibility(ir: InfrastructureIR) -> InfrastructureIR: reasons.append(f"{resource.resource_id} marked public without concrete path") # Sensitive + public without concrete path is always review. for resource in ir.resources: - if is_sensitive(resource.sensitivity) and resource.public_exposure and not _has_concrete(resource, ir.public_paths): + if ( + is_sensitive(resource.sensitivity) + and resource.public_exposure + and not _has_concrete(resource, ir.public_paths) + ): reasons.append(f"sensitive resource {resource.resource_id} lacks concrete exposure path") eligibility: Eligibility = "strict" if not reasons and ir.resources else "review" @@ -46,7 +50,5 @@ def sensitive_public_violations(ir: InfrastructureIR) -> list[InfraResourceIR]: return [ resource for resource in ir.resources - if is_sensitive(resource.sensitivity) - and resource.public_exposure - and _has_concrete(resource, ir.public_paths) + if is_sensitive(resource.sensitivity) and resource.public_exposure and _has_concrete(resource, ir.public_paths) ] diff --git a/ovk/compilers/infrastructure/terraform_plan.py b/ovk/compilers/infrastructure/terraform_plan.py index cc9da14..94bf1b0 100644 --- a/ovk/compilers/infrastructure/terraform_plan.py +++ b/ovk/compilers/infrastructure/terraform_plan.py @@ -2,6 +2,10 @@ Regex is never authoritative. Only known plan JSON fields are consumed; unknown shapes are marked unsupported and force review eligibility. + +Profile ``infrastructure.terraform.plan_recursive_v1`` expands +``planned_values`` / ``prior_state`` child modules recursively instead of +stopping at ``root_module``. """ from __future__ import annotations @@ -17,6 +21,70 @@ from ovk.compilers.infrastructure.reachability import evaluate_eligibility from ovk.compilers.infrastructure.sensitivity import sensitivity_from_tags +_SOURCE_PROFILE_ID = "infrastructure.terraform.plan_recursive_v1" + + +def _walk_module_resources( + module: dict[str, Any], + *, + module_address: str, + out: list[dict[str, Any]], + warnings: list[str], + depth: int = 0, + max_depth: int = 32, +) -> None: + """Recursively collect resources from a Terraform module tree.""" + if depth > max_depth: + warnings.append(f"module_depth_exceeded:{module_address or 'root'}") + return + resources = module.get("resources") + if isinstance(resources, list): + for item in resources: + if not isinstance(item, dict): + continue + address = item.get("address") + if not address: + name = item.get("name") or "unnamed" + rtype = item.get("type") or "unknown" + prefix = f"{module_address}." if module_address else "" + address = f"{prefix}{rtype}.{name}" + out.append( + { + "address": address, + "type": item.get("type"), + "change": {"after": item.get("values", {})}, + "module_address": module_address or "root", + } + ) + children = module.get("child_modules") + if not isinstance(children, list): + return + for child in children: + if not isinstance(child, dict): + continue + child_addr = str(child.get("address") or f"{module_address}.module.unknown") + _walk_module_resources( + child, + module_address=child_addr, + out=out, + warnings=warnings, + depth=depth + 1, + max_depth=max_depth, + ) + + +def expand_planned_values_recursively(plan: dict[str, Any], warnings: list[str]) -> list[dict[str, Any]]: + """Expand ``planned_values`` including nested ``child_modules``.""" + planned = plan.get("planned_values", {}) + if not isinstance(planned, dict): + return [] + root = planned.get("root_module", {}) + if not isinstance(root, dict): + return [] + out: list[dict[str, Any]] = [] + _walk_module_resources(root, module_address="", out=out, warnings=warnings) + return out + def compile_terraform_plan(plan: dict[str, Any]) -> InfrastructureIR: """Compile a terraform show -json document into infrastructure IR.""" @@ -37,21 +105,31 @@ def compile_terraform_plan(plan: dict[str, Any]) -> InfrastructureIR: if format_version is None: unsupported.append("missing_format_version") resource_changes = plan.get("resource_changes") + used_recursive_profile = False if resource_changes is None: - # Planned values fallback is accepted as partial. - planned = plan.get("planned_values", {}) - root = planned.get("root_module", {}) if isinstance(planned, dict) else {} - resource_changes = [] - for item in root.get("resources", []) if isinstance(root, dict) else []: - if isinstance(item, dict): - resource_changes.append( - { - "address": item.get("address") or item.get("name"), - "type": item.get("type"), - "change": {"after": item.get("values", {})}, - } - ) - warnings.append("resource_changes missing; used planned_values.root_module.resources") + # Planned values fallback with recursive child_modules expansion. + resource_changes = expand_planned_values_recursively(plan, warnings) + used_recursive_profile = True + warnings.append("resource_changes missing; used planned_values recursive module walk") + warnings.append(f"compiled_with_source_profile:{_SOURCE_PROFILE_ID}") + elif isinstance(resource_changes, list): + # Also surface nested planned_values modules as supplemental when present. + nested = expand_planned_values_recursively(plan, warnings) + if nested: + existing = { + str(item.get("address")) for item in resource_changes if isinstance(item, dict) and item.get("address") + } + added = 0 + for item in nested: + address = str(item.get("address") or "") + if address and address not in existing: + resource_changes.append(item) + existing.add(address) + added += 1 + if added: + used_recursive_profile = True + warnings.append(f"recursive_modules_added:{added}") + warnings.append(f"compiled_with_source_profile:{_SOURCE_PROFILE_ID}") if not isinstance(resource_changes, list): unsupported.append("resource_changes_not_list") @@ -81,6 +159,11 @@ def compile_terraform_plan(plan: dict[str, Any]) -> InfrastructureIR: elif after.get("internet_accessible") is True: paths = ["internet_accessible"] public = bool(paths) or after.get("public_exposure") is True + attributes: dict[str, Any] = {"format_version": format_version} + if change.get("module_address"): + attributes["module_address"] = change["module_address"] + if used_recursive_profile: + attributes["source_profile"] = _SOURCE_PROFILE_ID resources.append( InfraResourceIR( resource_id=address, @@ -89,7 +172,7 @@ def compile_terraform_plan(plan: dict[str, Any]) -> InfrastructureIR: sensitivity=sensitivity, public_exposure=public, exposure_paths=paths, - attributes={"format_version": format_version}, + attributes=attributes, ) ) diff --git a/ovk/core/adapter_runtime.py b/ovk/core/adapter_runtime.py index 7cde1f2..9e72b7a 100644 --- a/ovk/core/adapter_runtime.py +++ b/ovk/core/adapter_runtime.py @@ -1,34 +1,51 @@ -"""Execute compiled obligations through legacy lanes or the typed control plane. +"""Execute compiled obligations through lane adapters with routing metadata. + + + +Shadow mode runs the typed control plane beside legacy ``evaluate_lane``; + +legacy evidence remains authoritative unless a lane is policy-enforced. -Legacy mode may use the flat evidence cache. Shadow and enforced modes rely on -the routing-bound hardened control-plane cache so policy or routing changes can -never reuse evidence from another execution regime. """ from __future__ import annotations + from concurrent.futures import ThreadPoolExecutor + from pathlib import Path -from typing import Any, Callable + +from typing import Any, Mapping + from ovk.adapters.authorization import build_authorization_registry + from ovk.adapters.ci_secrets import build_ci_secrets_registry + from ovk.adapters.deployment import build_deployment_registry + from ovk.adapters.infrastructure import build_infrastructure_registry + from ovk.adapters.lane import build_default_lane_registry + from ovk.adapters.self_protection import build_self_protection_registry -from ovk.core.authorization_compiler import compile_authorization_obligation + from ovk.core.backend_control_plane import BackendControlPlane, compare_shadow_to_legacy + from ovk.core.bundle import content_digest -from ovk.core.ci_secrets_compiler import compile_ci_secrets_obligation -from ovk.core.deployment_compiler import compile_deployment_obligation + from ovk.core.evidence_from_execution import execution_record_to_evidence + from ovk.core.execution_budget import LocalSubprocessWorker, execution_budget_from_policy -from ovk.core.execution_models import ExecutionContext, VerificationObligation -from ovk.core.infrastructure_compiler import compile_infrastructure_obligation + +from ovk.core.execution_models import ExecutionContext, RoutingDecision, VerificationObligation + from ovk.core.models import VerificationEvidence + from ovk.core.multi_lane import evaluate_lane + from ovk.core.policy_config import resolve_routing_config, routing_enforced_for_lane + from ovk.core.result_cache import ( ControlPlaneResultCache, HardenedResultCache, @@ -36,10 +53,33 @@ get_cached_evidence, store_cached_evidence, ) -from ovk.core.router import RoutingConfig, route_obligation, routing_config_from_policy -from ovk.core.self_protection_compiler import compile_self_protection_obligation + +from ovk.core.router import route_obligation, routing_decision_to_legacy_dict + +from ovk.core.routing_pipeline import ( + AuthoritativeRoutingPlan, + ensure_authoritative_routing, + intent_id_for_obligation, + require_routing_decision, +) + +from ovk.core.self_protection_compiler import resolve_metadata_trusted + from ovk.core.shadow_obligation import build_shadow_obligation + +def _control_plane(*, cache_dir: Path | None = None) -> BackendControlPlane: + """Build an enforced/shadow control plane with hardened cache + worker.""" + + hardened = HardenedResultCache(cache_dir / "control-plane") if cache_dir is not None else HardenedResultCache() + + return BackendControlPlane( + cache=ControlPlaneResultCache(hardened), + worker=LocalSubprocessWorker(), + use_hardened_cache=True, + ) + + LANE_TO_INTENT = { "self_protection": "agent-cannot-disable-own-ci-gate", "authorization": "no-admin-route-bypass", @@ -48,22 +88,35 @@ "deployment": "no-skipped-approval-state", } -RegistryBuilder = Callable[[], Any] -ObligationCompiler = Callable[..., VerificationObligation] +_LANE_REGISTRY_BUILDERS = { + "authorization": build_authorization_registry, + "self_protection": build_self_protection_registry, + "infrastructure": build_infrastructure_registry, + "ci_secrets": build_ci_secrets_registry, + "deployment": build_deployment_registry, +} -def _control_plane(*, cache_dir: Path | None = None) -> BackendControlPlane: - """Build a control plane with a routing-bound hardened cache and worker.""" - hardened = ( - HardenedResultCache(cache_dir / "control-plane") - if cache_dir is not None - else HardenedResultCache() - ) - return BackendControlPlane( - cache=ControlPlaneResultCache(hardened), - worker=LocalSubprocessWorker(), - use_hardened_cache=True, - ) + +def _routing_metadata( + routing: RoutingDecision | Mapping[str, Any] | None, + *, + intent_id: str, + routing_enforced: bool, +) -> dict[str, Any] | None: + + if routing is None: + return None + + if isinstance(routing, RoutingDecision): + payload = routing_decision_to_legacy_dict(routing, intent_id=intent_id) + + else: + payload = dict(routing) + + payload["routing_enforced"] = routing_enforced + + return payload def _attach_execution_metadata( @@ -71,95 +124,73 @@ def _attach_execution_metadata( *, lane: str, data: dict[str, Any], - routing: dict[str, Any] | None, + routing: RoutingDecision | Mapping[str, Any] | None, shadow_comparison: dict[str, Any] | None = None, intent_id: str | None = None, job_id: str | None = None, input_format: str | None = None, + routing_enforced: bool = False, ) -> VerificationEvidence: - """Attach compatibility metadata without contradicting authoritative v2 routing.""" - resolved_intent = intent_id or str( - (routing or {}).get("intent_id") or LANE_TO_INTENT.get(lane, lane) + """Record routing, input digest, and obligation-scoped evidence identity.""" + + resolved_intent = ( + intent_id + or str((routing or {}).get("intent_id") if isinstance(routing, dict) else None) + or LANE_TO_INTENT.get(lane, lane) ) + resolved_format = input_format or "infra" + + identity = { + "intent_id": resolved_intent, + "lane": lane, + "input": data, + "input_format": resolved_format, + "job_id": job_id, + } + input_digest = content_digest({"lane": lane, "input": data}) + + evidence_suffix = content_digest(identity)[:12] + artifacts = list(evidence.generated_artifacts) - if not any( - item.get("kind") == "input_digest" and item.get("digest") == input_digest - for item in artifacts - if isinstance(item, dict) - ): - artifacts.append({"kind": "input_digest", "digest": input_digest, "lane": lane}) - - # Enforced evidence already contains the authoritative typed routing record. - # A legacy capability-router artifact would create two conflicting stories. - if routing is not None and evidence.routing_enforced is not True: + + artifacts.append({"kind": "input_digest", "digest": input_digest, "lane": lane}) + + metadata = _routing_metadata(routing, intent_id=resolved_intent, routing_enforced=routing_enforced) + + if metadata is not None: artifacts.append( { "kind": "backend_routing", - "intent_id": routing.get("intent_id"), - "selected": routing.get("selected", []), - "rejected": routing.get("rejected", []), - "routing_id": routing.get("routing_id"), - "routing_enforced": False, + "intent_id": metadata.get("intent_id", resolved_intent), + "selected": metadata.get("selected", []), + "rejected": metadata.get("rejected", []), + "routing_id": metadata.get("routing_id") or evidence.routing_id, + "routing_enforced": routing_enforced, "executed_backends": [claim.backend for claim in evidence.backend_claims], } ) + if shadow_comparison is not None: artifacts.append(shadow_comparison) - if evidence.routing_enforced is True: - return evidence.model_copy(update={"generated_artifacts": artifacts}) - - identity = { - "intent_id": resolved_intent, - "lane": lane, - "input": data, - "input_format": resolved_format, - "job_id": job_id, - } - evidence_suffix = content_digest(identity)[:12] return evidence.model_copy( update={ "evidence_id": f"{evidence.evidence_id}-{evidence_suffix}", "generated_artifacts": artifacts, + "routing_id": evidence.routing_id or (metadata or {}).get("routing_id"), } ) def _legacy_status_and_recommendation(evidence: VerificationEvidence) -> tuple[str, str]: + status = evidence.backend_claims[0].status.value if evidence.backend_claims else "unknown" - recommendation = str( - evidence.decision.get("merge_recommendation", "require_human_review") - ) - return status, recommendation + recommendation = str(evidence.decision.get("merge_recommendation", "require_human_review")) -def _effective_policy_digest( - policy: dict[str, Any] | None, - policy_path: Path | None, -) -> str: - lane_policy: dict[str, Any] | str | None = None - if policy_path is not None and policy_path.is_file(): - try: - lane_policy = policy_path.read_text(encoding="utf-8") - except OSError: - lane_policy = "unreadable" - return content_digest({"repository_policy": policy or {}, "lane_policy": lane_policy}) - - -def _routing_config_for_enforced(policy: dict[str, Any] | None, lane: str) -> RoutingConfig: - current = routing_config_from_policy(policy) - return RoutingConfig( - mode="enforced", - strategy=current.strategy, - aggregation=current.aggregation, - max_selected_backends=current.max_selected_backends, - prefer_deterministic=current.prefer_deterministic, - allow_fallback=current.allow_fallback, - accept_partial_primary=current.accept_partial_primary, - enforced_lanes=frozenset({lane}), - ) + return status, recommendation def _run_shadow_path( @@ -171,11 +202,12 @@ def _run_shadow_path( base_sha: str | None, intent_id: str, policy: dict[str, Any] | None, - cache_dir: Path | None, -) -> dict[str, Any]: - """Execute the typed lane-wrapper control plane for comparison only.""" +) -> dict[str, Any] | None: + """Execute the typed control plane for comparison; never raises to legacy.""" + try: registry = build_default_lane_registry() + obligation = build_shadow_obligation( lane=lane, data=data, @@ -184,21 +216,26 @@ def _run_shadow_path( base_sha=base_sha, intent_id=intent_id, ) + budget = execution_budget_from_policy(policy) + context = ExecutionContext( subject=obligation.subject, budget=budget, policy_digest=obligation.policy_digest, metadata={"shadow": True}, ) + routing = route_obligation(obligation, registry, context=context, policy=policy) - record = _control_plane(cache_dir=cache_dir).execute( - obligation, - routing, - registry=registry, - ) - return {"record": record, "routing": routing} - except Exception as exc: # noqa: BLE001 - shadow cannot affect legacy authority + + record = _control_plane().execute(obligation, routing, registry=registry) + + return { + "record": record, + "routing": routing, + } + + except Exception as exc: # noqa: BLE001 - shadow must not affect legacy authority return { "error": { "category": type(exc).__name__, @@ -207,201 +244,135 @@ def _run_shadow_path( } -def _compile_enforced_obligation( +def _run_enforced_with_routing( *, lane: str, data: dict[str, Any], - repo: str, - head_sha: str, - base_sha: str | None, + routing: RoutingDecision, + typed_obligation: VerificationObligation, policy: dict[str, Any] | None, - policy_digest: str, -) -> tuple[Any, VerificationObligation, dict[str, str]]: - if lane == "self_protection": - registry = build_self_protection_registry() - metadata_trusted = True - if isinstance(policy, dict): - trust = policy.get("trust", {}) - if isinstance(trust, dict) and "metadata_trusted" in trust: - metadata_trusted = bool(trust.get("metadata_trusted")) - routing = policy.get("routing", {}) - if isinstance(routing, dict) and "metadata_trusted" in routing: - metadata_trusted = bool(routing.get("metadata_trusted")) - obligation = compile_self_protection_obligation( - data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - policy_digest=policy_digest, - metadata_trusted=metadata_trusted, - ) - actor = data.get("actor") if isinstance(data.get("actor"), dict) else {} - metadata = { - "author_type": str(actor.get("type", data.get("author_type", "unknown"))), - "agent": str(actor.get("id", data.get("agent", "unknown"))), - "task": str(data.get("task", "unknown")), - "metadata_trusted": str(metadata_trusted), - } - return registry, obligation, metadata + schema_version: str, +) -> VerificationEvidence: + """Execute one enforced lane using a pre-computed immutable routing decision.""" + + registry_builder = _LANE_REGISTRY_BUILDERS.get(lane) + + if registry_builder is None: + raise RuntimeError(f"no enforced registry for lane {lane!r}") - builders: dict[str, tuple[RegistryBuilder, ObligationCompiler]] = { - "authorization": (build_authorization_registry, compile_authorization_obligation), - "infrastructure": (build_infrastructure_registry, compile_infrastructure_obligation), - "ci_secrets": (build_ci_secrets_registry, compile_ci_secrets_obligation), - "deployment": (build_deployment_registry, compile_deployment_obligation), - } - registry_builder, compiler = builders[lane] registry = registry_builder() - obligation = compiler( - data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - policy_digest=policy_digest, - policy=policy, - ) - metadata = { - "author_type": str(data.get("author_type", "unknown")), - "agent": str(data.get("agent", "unknown")), - "task": str(data.get("task", "unknown")), - } - return registry, obligation, metadata + record = _control_plane().execute(typed_obligation, routing, registry=registry) -def _run_enforced_lane( - *, - lane: str, - data: dict[str, Any], - repo: str, - head_sha: str, - base_sha: str | None, - policy: dict[str, Any] | None, - policy_digest: str, - cache_dir: Path | None, -) -> VerificationEvidence: - registry, obligation, metadata = _compile_enforced_obligation( - lane=lane, - data=data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - policy=policy, - policy_digest=policy_digest, - ) - budget = execution_budget_from_policy(policy) - context = ExecutionContext( - subject=obligation.subject, - budget=budget, - policy_digest=obligation.policy_digest, - metadata={"enforced": True, "lane": lane}, - ) - routing = route_obligation( - obligation, - registry, - context=context, - config=_routing_config_for_enforced(policy, lane), - policy=policy, - ) - record = _control_plane(cache_dir=cache_dir).execute( - obligation, - routing, - registry=registry, - ) evidence = execution_record_to_evidence( record, - author_type=metadata["author_type"], - agent=metadata["agent"], - task=metadata["task"], + author_type=str((data.get("actor") or {}).get("type", data.get("author_type", "unknown"))), + agent=str((data.get("actor") or {}).get("id", data.get("agent", "unknown"))), + task=str(data.get("task", "unknown")), routing_enforced=True, - schema_version="ovk.evidence.v2", + schema_version=schema_version, ) - if ( - lane == "self_protection" - and metadata.get("metadata_trusted") == "False" - and evidence.decision.get("merge_recommendation") == "allow" - ): - evidence = evidence.model_copy( - update={ - "decision": { - **evidence.decision, - "merge_recommendation": "require_human_review", - "human_review_required": True, - "reason": "untrusted metadata cannot authorize allow under enforcement", - "fallback_accepted": False, + + if lane == "self_protection": + metadata_trusted = resolve_metadata_trusted(policy) + + if not metadata_trusted and evidence.decision.get("merge_recommendation") == "allow": + evidence = evidence.model_copy( + update={ + "decision": { + **evidence.decision, + "merge_recommendation": "require_human_review", + "human_review_required": True, + "reason": "untrusted metadata cannot authorize allow under enforcement", + "fallback_accepted": False, + } } - } - ) + ) + return evidence def _evaluate_obligation( obligation: dict[str, Any], *, - routing_by_intent: dict[str, dict[str, Any]], + routing_plan: AuthoritativeRoutingPlan, repo: str, head_sha: str, base_sha: str | None, cache_dir: Path | None, use_cache: bool, policy: dict[str, Any] | None = None, + evidence_schema_version: str = "ovk.evidence.v3", ) -> VerificationEvidence: + lane = str(obligation["lane"]) + data = obligation["input"] - intent_id = str(obligation.get("intent_id") or LANE_TO_INTENT.get(lane, lane)) + + intent_id = intent_id_for_obligation(obligation) + input_format = str(obligation.get("input_format", "infra")) - policy_path = Path(obligation["policy_path"]) if obligation.get("policy_path") else None - routing_config = resolve_routing_config(policy) - enforced = lane in LANE_TO_INTENT and routing_enforced_for_lane(policy, lane) - legacy_cache_allowed = routing_config.mode == "legacy" and not enforced - policy_digest = _effective_policy_digest(policy, policy_path) key = cache_key( lane, data, - policy_digest=policy_digest, - subject={ - "repo": repo, - "head_sha": head_sha, - **({"base_sha": base_sha} if base_sha else {}), - }, - execution_fingerprint={ - "intent_id": intent_id, - "input_format": input_format, - "routing_mode": routing_config.mode, - }, + subject={"repo": repo, "head_sha": head_sha, **({"base_sha": base_sha} if base_sha else {})}, + execution_fingerprint={"intent_id": intent_id, "input_format": input_format}, ) - if legacy_cache_allowed and use_cache and cache_dir is not None: + + precomputed_routing = routing_plan.routing_by_intent.get(intent_id) + + precomputed_typed = routing_plan.typed_obligations.get(intent_id) + + if use_cache and cache_dir is not None: cached = get_cached_evidence(cache_dir, key) + if cached is not None: + evidence = VerificationEvidence.model_validate(cached) + return _attach_execution_metadata( - VerificationEvidence.model_validate(cached), + evidence, lane=lane, data=data, - routing=routing_by_intent.get(intent_id), + routing=precomputed_routing, intent_id=intent_id, job_id=obligation.get("job_id"), input_format=input_format, + routing_enforced=routing_enforced_for_lane(policy, lane), ) - if enforced: - evidence = _run_enforced_lane( + if routing_enforced_for_lane(policy, lane): + routing_decision = require_routing_decision( + precomputed_routing, + intent_id=intent_id, + lane=lane, + policy=policy, + ) + + if precomputed_typed is None: + raise RuntimeError(f"missing typed obligation for enforced intent {intent_id!r}") + + evidence = _run_enforced_with_routing( lane=lane, data=data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, + routing=routing_decision, + typed_obligation=precomputed_typed, policy=policy, - policy_digest=policy_digest, - cache_dir=cache_dir if use_cache else None, + schema_version=evidence_schema_version, ) + + if use_cache and cache_dir is not None: + store_cached_evidence(cache_dir, key, evidence.model_dump(mode="json")) + return _attach_execution_metadata( evidence, lane=lane, data=data, - routing=None, + routing=routing_decision, intent_id=intent_id, job_id=obligation.get("job_id"), input_format=input_format, + routing_enforced=True, ) evidence = evaluate_lane( @@ -411,11 +382,14 @@ def _evaluate_obligation( head_sha=head_sha, base_sha=base_sha, input_format=input_format, - policy_path=policy_path, + policy_path=Path(obligation["policy_path"]) if obligation.get("policy_path") else None, ) shadow_comparison: dict[str, Any] | None = None - if routing_config.mode in {"shadow", "enforced"} and lane in LANE_TO_INTENT: + + routing_config = resolve_routing_config(policy) + + if routing_config.mode in {"shadow", "enforced"} and lane in _LANE_REGISTRY_BUILDERS: shadow = _run_shadow_path( lane=lane, data=data, @@ -424,43 +398,49 @@ def _evaluate_obligation( base_sha=base_sha, intent_id=intent_id, policy=policy, - cache_dir=cache_dir if use_cache else None, ) - if "record" in shadow: + + if shadow and "record" in shadow: legacy_status, legacy_recommendation = _legacy_status_and_recommendation(evidence) + shadow_comparison = compare_shadow_to_legacy( shadow=shadow["record"], legacy_status=legacy_status, legacy_recommendation=legacy_recommendation, ) + shadow_comparison["routing_mode"] = routing_config.mode + shadow_comparison["legacy_authoritative"] = True - else: + + elif shadow and "error" in shadow: shadow_comparison = { "kind": "shadow_comparison", "agreement": False, - "error": shadow.get("error", {"message": "shadow execution failed"}), + "error": shadow["error"], "legacy_authoritative": True, "routing_mode": routing_config.mode, } - if legacy_cache_allowed and use_cache and cache_dir is not None: + if use_cache and cache_dir is not None: store_cached_evidence(cache_dir, key, evidence.model_dump(mode="json")) + return _attach_execution_metadata( evidence, lane=lane, data=data, - routing=routing_by_intent.get(intent_id), + routing=precomputed_routing, shadow_comparison=shadow_comparison, intent_id=intent_id, job_id=obligation.get("job_id"), input_format=input_format, + routing_enforced=False, ) def execute_obligations( obligations: list[dict[str, Any]], - routing_by_intent: dict[str, dict[str, Any]], + routing_by_intent: Mapping[str, RoutingDecision | Mapping[str, Any] | None] | None = None, *, repo: str, head_sha: str, @@ -469,38 +449,53 @@ def execute_obligations( use_cache: bool = True, parallel: bool = True, policy: dict[str, Any] | None = None, + evidence_schema_version: str = "ovk.evidence.v3", ) -> list[VerificationEvidence]: - """Evaluate obligations while preserving deterministic submission order.""" + """Evaluate obligations using one authoritative routing decision per intent.""" + if not obligations: return [] + + routing_plan = ensure_authoritative_routing( + obligations, + routing_by_intent, + policy=policy, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + ) + if parallel and len(obligations) > 1: with ThreadPoolExecutor(max_workers=min(len(obligations), 5)) as pool: futures = [ pool.submit( _evaluate_obligation, obligation, - routing_by_intent=routing_by_intent, + routing_plan=routing_plan, repo=repo, head_sha=head_sha, base_sha=base_sha, cache_dir=cache_dir, use_cache=use_cache, policy=policy, + evidence_schema_version=evidence_schema_version, ) for obligation in obligations ] + return [future.result() for future in futures] return [ _evaluate_obligation( obligation, - routing_by_intent=routing_by_intent, + routing_plan=routing_plan, repo=repo, head_sha=head_sha, base_sha=base_sha, cache_dir=cache_dir, use_cache=use_cache, policy=policy, + evidence_schema_version=evidence_schema_version, ) for obligation in obligations ] diff --git a/ovk/core/attestation.py b/ovk/core/attestation.py index c7a14d8..a6cb846 100644 --- a/ovk/core/attestation.py +++ b/ovk/core/attestation.py @@ -40,6 +40,7 @@ def bundle_to_statement(bundle: EvidenceBundle) -> dict[str, Any]: "decision": evidence.decision, "obligation_id": evidence.obligation_id, "routing_id": evidence.routing_id, + "material_set_digest": evidence.material_set_digest, "compiler": evidence.compiler, "coverage": evidence.coverage, "materials": evidence.materials, diff --git a/ovk/core/attestation_binding.py b/ovk/core/attestation_binding.py index 176f5fe..39c9196 100644 --- a/ovk/core/attestation_binding.py +++ b/ovk/core/attestation_binding.py @@ -68,6 +68,14 @@ def verify_bundle_statement_binding(bundle: EvidenceBundle, statement: dict[str, message="evidence and attestation routing IDs disagree (OVK-INV-020)", ) ) + stated_material_set = evidence_items[index].get("material_set_digest") + if evidence.material_set_digest and stated_material_set != evidence.material_set_digest: + issues.append( + EvidenceInvariantIssue( + path=f"predicate.verification.evidence[{index}].material_set_digest", + message="evidence and attestation material_set_digest disagree (OVK-INV-021)", + ) + ) return issues diff --git a/ovk/core/backend_aggregation.py b/ovk/core/backend_aggregation.py index 9a67885..df5ae16 100644 --- a/ovk/core/backend_aggregation.py +++ b/ovk/core/backend_aggregation.py @@ -8,11 +8,26 @@ from dataclasses import dataclass from typing import Any, Sequence -from ovk.core.execution_models import BackendSelection, NormalizedBackendResult +from ovk.core.execution_models import ( + BackendSelection, + ExecutionAttempt, + FallbackPolicy, + NormalizedBackendResult, + TerminationKind, +) from ovk.core.models import MergeRecommendation, VerificationStatus AGGREGATION_FAIL_DOMINANT_V1 = "ovk.aggregate.fail_dominant.v1" +FALLBACK_BLOCKING_TERMINATIONS: frozenset[TerminationKind] = frozenset( + { + "timeout", + "tool_error", + "invalid_output", + "resource_exhausted", + } +) + @dataclass(frozen=True) class AggregationOutcome: @@ -24,6 +39,57 @@ class AggregationOutcome: disagreement: dict[str, Any] | None = None warnings: tuple[str, ...] = () quality_error: bool = False + fallback_used: bool = False + fallback_accepted: bool = False + fallback_cause: str | None = None + + +def evaluate_fallback_acceptance( + *, + policy: FallbackPolicy, + selected: Sequence[BackendSelection], + attempts: Sequence[ExecutionAttempt], + results: Sequence[NormalizedBackendResult], + acceptable_guarantees: Sequence[str] | None = None, +) -> tuple[bool, bool, str | None]: + """Decide whether weaker fallback evidence may satisfy guarantee requirements (INV-017).""" + acceptable = set(acceptable_guarantees or []) + if not acceptable: + return False, False, None + + attempts_by_backend = {item.backend: item for item in attempts} + fallback_used = False + fallback_cause: str | None = None + + for selection in selected: + if not selection.required: + continue + result = next((item for item in results if item.backend == selection.backend), None) + attempt = attempts_by_backend.get(selection.backend) + if result is None or attempt is None: + continue + if result.status != VerificationStatus.PASS: + continue + if result.guarantee_type in acceptable: + continue + + fallback_used = True + fallback_cause = attempt.termination + + if attempt.termination in FALLBACK_BLOCKING_TERMINATIONS: + return True, False, attempt.termination + if not policy.allow_fallback: + return True, False, attempt.termination + if policy.outcome_for_termination(attempt.termination) in {"fail", "error"}: + return True, False, attempt.termination + if policy.fallback_backends and selection.backend not in policy.fallback_backends: + return True, False, attempt.termination + if policy.acceptable_fallback_guarantees and result.guarantee_type not in policy.acceptable_fallback_guarantees: + return True, False, attempt.termination + + if fallback_used: + return True, True, fallback_cause + return False, False, None def build_disagreement_artifact( @@ -58,7 +124,9 @@ def aggregate_fail_dominant_v1( selected: Sequence[BackendSelection], results: Sequence[NormalizedBackendResult], acceptable_guarantees: Sequence[str] | None = None, - fallback_accepted: bool = False, + fallback_accepted: bool | None = None, + fallback_policy: FallbackPolicy | None = None, + attempts: Sequence[ExecutionAttempt] | None = None, ) -> AggregationOutcome: """Apply the fail-dominant aggregation decision table. @@ -75,6 +143,20 @@ def aggregate_fail_dominant_v1( * optional unknown/error warns without invalidating required pass * optional pass cannot upgrade required unknown """ + policy = fallback_policy or FallbackPolicy() + if attempts is not None: + fallback_used, resolved_fallback_accepted, fallback_cause = evaluate_fallback_acceptance( + policy=policy, + selected=selected, + attempts=attempts, + results=results, + acceptable_guarantees=acceptable_guarantees, + ) + else: + fallback_used = False + fallback_cause = None + resolved_fallback_accepted = bool(fallback_accepted) + selected_required = [item for item in selected if item.required] selected_optional = [item for item in selected if not item.required] by_backend = _statuses_by_backend(results) @@ -87,10 +169,7 @@ def aggregate_fail_dominant_v1( return AggregationOutcome( status=VerificationStatus.UNKNOWN, merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, - reason=( - "selected and executed backend sets differ; " - f"missing={missing}; unexpected={unexpected}" - ), + reason=(f"selected and executed backend sets differ; missing={missing}; unexpected={unexpected}"), quality_error=True, ) @@ -121,6 +200,9 @@ def aggregate_fail_dominant_v1( merge_recommendation=MergeRecommendation.BLOCK, reason="optional corroborator reported fail", disagreement=disagreement, + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) if any(item.status == VerificationStatus.FAIL for item in required_results): @@ -135,6 +217,9 @@ def aggregate_fail_dominant_v1( merge_recommendation=MergeRecommendation.BLOCK, reason="required backend reported fail", disagreement=disagreement, + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) non_pass = { @@ -145,14 +230,15 @@ def aggregate_fail_dominant_v1( if any(item.status in non_pass for item in required_results): for item in optional_results: if item.status == VerificationStatus.PASS: - warnings.append( - f"optional backend {item.backend} passed but cannot upgrade required unknown/error" - ) + warnings.append(f"optional backend {item.backend} passed but cannot upgrade required unknown/error") return AggregationOutcome( status=VerificationStatus.UNKNOWN, merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, reason="required backend reported unknown, error, or skipped", warnings=tuple(warnings), + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) if not required_results and not selected_required: @@ -161,19 +247,24 @@ def aggregate_fail_dominant_v1( status=VerificationStatus.UNKNOWN, merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, reason="no required backends were selected", + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) # Check guarantees / fallback acceptance for required passes. acceptable = set(acceptable_guarantees or []) for item in required_results: - if acceptable and item.guarantee_type not in acceptable and not fallback_accepted: + if acceptable and item.guarantee_type not in acceptable and not resolved_fallback_accepted: return AggregationOutcome( status=VerificationStatus.UNKNOWN, merge_recommendation=MergeRecommendation.REQUIRE_STRONGER_CHECK, reason=( - f"required result from {item.backend} uses guarantee " - f"{item.guarantee_type!r} outside acceptable set" + f"required result from {item.backend} uses guarantee {item.guarantee_type!r} outside acceptable set" ), + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) for item in optional_results: @@ -185,6 +276,9 @@ def aggregate_fail_dominant_v1( merge_recommendation=MergeRecommendation.ALLOW, reason="every required backend passed with acceptable guarantees", warnings=tuple(warnings), + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) @@ -195,7 +289,9 @@ def aggregate_results( results: Sequence[NormalizedBackendResult], policy: str = AGGREGATION_FAIL_DOMINANT_V1, acceptable_guarantees: Sequence[str] | None = None, - fallback_accepted: bool = False, + fallback_accepted: bool | None = None, + fallback_policy: FallbackPolicy | None = None, + attempts: Sequence[ExecutionAttempt] | None = None, ) -> AggregationOutcome: """Dispatch to a versioned aggregation policy.""" if policy != AGGREGATION_FAIL_DOMINANT_V1: @@ -206,4 +302,6 @@ def aggregate_results( results=results, acceptable_guarantees=acceptable_guarantees, fallback_accepted=fallback_accepted, + fallback_policy=fallback_policy, + attempts=attempts, ) diff --git a/ovk/core/backend_control_plane.py b/ovk/core/backend_control_plane.py index a3a5fea..036192e 100644 --- a/ovk/core/backend_control_plane.py +++ b/ovk/core/backend_control_plane.py @@ -19,11 +19,11 @@ from ovk.core.backend_aggregation import aggregate_results from ovk.core.backend_registry import BackendRegistry, BackendRegistryError -from ovk.core.bundle import content_digest from ovk.core.execution_budget import BackendWorker, LocalSubprocessWorker from ovk.core.execution_models import ( BackendEnvironmentFingerprint, BackendObligation, + CachedBackendExecution, ExecutionAttempt, ExecutionBudget, NormalizedBackendResult, @@ -39,11 +39,11 @@ class ResultCache(Protocol): - """Minimal cache protocol used by the control plane.""" + """Minimal cache protocol used by the control plane (ovk.cache.v3).""" - def get(self, key: str) -> NormalizedBackendResult | None: ... + def get(self, key: str) -> CachedBackendExecution | None: ... - def put(self, key: str, value: NormalizedBackendResult, *, meta: dict[str, Any]) -> None: ... + def put(self, key: str, value: CachedBackendExecution, *, meta: dict[str, Any]) -> None: ... _CACHE_UNSET = object() @@ -136,6 +136,37 @@ def _error_raw( return raw.model_copy(update=compute_raw_execution_digests(raw)) +def _compiler_contract_error_raw( + *, + backend: str, + backend_obligation_id: str, + message: str, + started_at: str, + started_perf: float, +) -> RawBackendExecution: + finished_at = _utc_now_iso() + raw = RawBackendExecution( + backend=backend, + backend_obligation_id=backend_obligation_id, + termination="invalid_output", + native_execution=False, + exit_code=1, + stderr=message, + raw_result={ + "status": "error", + "error": { + "category": "compiler_contract", + "message": message, + "stage": "compile", + }, + }, + started_at=started_at, + finished_at=finished_at, + duration_ms=(time.perf_counter() - started_perf) * 1000.0, + ) + return raw.model_copy(update=compute_raw_execution_digests(raw)) + + def _attempt_from_raw( *, raw: RawBackendExecution, @@ -235,7 +266,8 @@ def execute( results=results, policy=routing.aggregation_policy, acceptable_guarantees=obligation.acceptable_guarantees, - fallback_accepted=routing.fallback_policy.allow_fallback, + fallback_policy=routing.fallback_policy, + attempts=attempts, ) open_obligations: list[dict[str, Any]] = [] if outcome.disagreement is not None: @@ -261,6 +293,9 @@ def execute( merge_recommendation=outcome.merge_recommendation, aggregation_reason=outcome.reason, open_obligations=open_obligations, + fallback_used=outcome.fallback_used, + fallback_accepted=outcome.fallback_accepted, + fallback_cause=outcome.fallback_cause, ) def _execute_one( @@ -282,18 +317,43 @@ def _execute_one( adapter = registry.require(selection_backend) if adapter.backend_id != selection_backend: raise BackendRegistryError( - f"adapter identity mismatch: expected {selection_backend}, " - f"got {adapter.backend_id}" + f"adapter identity mismatch: expected {selection_backend}, got {adapter.backend_id}" ) compiled = adapter.compile(obligation, routing) if compiled.backend != selection_backend: raise BackendRegistryError( - f"compiled backend {compiled.backend!r} does not match selection " - f"{selection_backend!r}" + f"compiled backend {compiled.backend!r} does not match selection {selection_backend!r}" ) if compiled.expected_guarantee != expected_guarantee and expected_guarantee: - # Record expected guarantee from routing when adapter differs only by alias. - compiled = compiled.model_copy(update={"expected_guarantee": expected_guarantee}) + message = ( + "compiler contract violation: compiled guarantee " + f"{compiled.expected_guarantee!r} does not match routing " + f"expected guarantee {expected_guarantee!r}" + ) + raw = _compiler_contract_error_raw( + backend=selection_backend, + backend_obligation_id=compiled.backend_obligation_id, + message=message, + started_at=started_at, + started_perf=started_perf, + ) + attempt = _attempt_from_raw(raw=raw, required=required) + result = NormalizedBackendResult( + attempt_id=attempt.attempt_id, + backend=selection_backend, + status=VerificationStatus.UNKNOWN, + guarantee_type=expected_guarantee, + assumptions=[], + limits=["compiler guarantee mismatch; execution skipped"], + counterexamples=[ + { + "summary": message, + "failure_mode": "compiler_contract_violation", + } + ], + generated_artifacts=[], + ) + return attempt, result, compiled fingerprint = adapter.fingerprint(compiled) components = control_plane_cache_components( @@ -314,46 +374,42 @@ def _execute_one( bind(key, components) cached = cache.get(key) if cached is not None: - attempt = ExecutionAttempt( - attempt_id="pending", - backend_obligation_id=compiled.backend_obligation_id, - backend=selection_backend, - required=required, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started_perf) * 1000.0, - termination="completed", - native_execution=fingerprint.native_available, - tool_version=fingerprint.tool_version, - tool_digest=fingerprint.tool_digest, - worker_image_digest=fingerprint.worker_image_digest, - raw_result_digest=content_digest({"cache_hit": True, "key": key}), - ) - attempt = attempt.model_copy(update={"attempt_id": compute_attempt_id(attempt)}) - result = cached.model_copy(update={"attempt_id": attempt.attempt_id}) - return attempt, result, compiled + # Replay stored provenance; never re-infer native_execution. + stored_attempt = cached.attempt + result = cached.normalized_result.model_copy(update={"attempt_id": stored_attempt.attempt_id}) + return stored_attempt, result, compiled raw = self._run_adapter(adapter, compiled, budget) normalized = adapter.normalize(raw, compiled) attempt = _attempt_from_raw(raw=raw, required=required) result = normalized.model_copy(update={"attempt_id": attempt.attempt_id}) if cache is not None: + cached_exec = CachedBackendExecution( + attempt=attempt, + native_execution=attempt.native_execution, + tool_version=attempt.tool_version, + tool_digest=attempt.tool_digest, + termination=attempt.termination, + exit_code=attempt.exit_code, + raw_result_digest=attempt.raw_result_digest, + environment_fingerprint=fingerprint.environment_digest, + normalized_result=result, + ) cache.put( key, - result, + cached_exec, meta={ "environment_digest": fingerprint.environment_digest, "raw_result_digest": raw.raw_result_digest, "created_at": _utc_now_iso(), + "cache_schema_version": "ovk.cache.v3", }, ) return attempt, result, compiled except Exception as exc: # noqa: BLE001 - isolate failures at backend boundary raw = _error_raw( backend=selection_backend, - backend_obligation_id=( - compiled.backend_obligation_id if compiled is not None else "uncompiled" - ), + backend_obligation_id=(compiled.backend_obligation_id if compiled is not None else "uncompiled"), stage="execute", exc=exc, started_at=started_at, @@ -377,7 +433,6 @@ def _execute_one( ) return attempt, result, compiled - def _run_adapter(self, adapter: Any, compiled: BackendObligation, budget: ExecutionBudget) -> RawBackendExecution: """Invoke adapter.run, threading the worker when the adapter accepts it.""" run = adapter.run diff --git a/ovk/core/backend_ids.py b/ovk/core/backend_ids.py index ba31555..3787e52 100644 --- a/ovk/core/backend_ids.py +++ b/ovk/core/backend_ids.py @@ -46,15 +46,9 @@ "lane-deployment", } ), - "self_protection": frozenset( - {"opa-native", "self-protection-deterministic", "lane-self-protection"} - ), - "authorization": frozenset( - {"z3-native", "authorization-deterministic", "lane-authorization"} - ), - "infrastructure": frozenset( - {"infrastructure-deterministic", "lane-infrastructure"} - ), + "self_protection": frozenset({"opa-native", "self-protection-deterministic", "lane-self-protection"}), + "authorization": frozenset({"z3-native", "authorization-deterministic", "lane-authorization"}), + "infrastructure": frozenset({"infrastructure-deterministic", "lane-infrastructure"}), "ci_secrets": frozenset({"ci-secrets-deterministic", "lane-ci-secrets"}), "deployment": frozenset({"deployment-deterministic", "lane-deployment"}), } diff --git a/ovk/core/backend_registry.py b/ovk/core/backend_registry.py index 5840247..adc31c1 100644 --- a/ovk/core/backend_registry.py +++ b/ovk/core/backend_registry.py @@ -34,8 +34,7 @@ def _validate_capability_manifest(manifest: BackendCapabilityManifest) -> None: report = validate_against_schema(payload, schema) if not report.valid: issues = "; ".join( - f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" - for issue in report.issues + f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" for issue in report.issues ) raise BackendRegistryError(f"capability manifest failed schema validation: {issues}") @@ -68,15 +67,11 @@ def register(self, adapter: BackendAdapter) -> None: raise BackendRegistryError(f"duplicate backend registration: {backend_id}") adapter_key = (adapter_id, adapter_version) if adapter_key in self._adapter_keys: - raise BackendRegistryError( - f"duplicate adapter identity: {adapter_id}@{adapter_version}" - ) + raise BackendRegistryError(f"duplicate adapter identity: {adapter_id}@{adapter_version}") manifest = adapter.manifest() if not isinstance(manifest, BackendCapabilityManifest): - raise BackendRegistryError( - f"adapter {backend_id!r} manifest() must return BackendCapabilityManifest" - ) + raise BackendRegistryError(f"adapter {backend_id!r} manifest() must return BackendCapabilityManifest") tool = manifest.tool if tool.adapter != adapter_id: raise BackendRegistryError( @@ -93,17 +88,13 @@ def register(self, adapter: BackendAdapter) -> None: if not manifest.supported_domains: raise BackendRegistryError(f"adapter {backend_id!r} must declare supported_domains") if not manifest.supported_property_kinds: - raise BackendRegistryError( - f"adapter {backend_id!r} must declare supported_property_kinds" - ) + raise BackendRegistryError(f"adapter {backend_id!r} must declare supported_property_kinds") _validate_capability_manifest(manifest) # Require compile/run surface present (Protocol methods). for method_name in ("can_handle", "compile", "fingerprint", "run", "normalize", "explain"): if not callable(getattr(adapter, method_name, None)): - raise BackendRegistryError( - f"adapter {backend_id!r} is missing required method {method_name}" - ) + raise BackendRegistryError(f"adapter {backend_id!r} is missing required method {method_name}") self._by_backend[backend_id] = adapter self._adapter_keys.add(adapter_key) @@ -142,8 +133,7 @@ def candidates( assessment = adapter.can_handle(obligation, context) if assessment.backend != adapter.backend_id: raise BackendRegistryError( - f"adapter {adapter.backend_id!r} returned assessment for " - f"backend {assessment.backend!r}" + f"adapter {adapter.backend_id!r} returned assessment for backend {assessment.backend!r}" ) assessments.append(assessment) return sorted( diff --git a/ovk/core/bundle.py b/ovk/core/bundle.py index ca101d9..489a83f 100644 --- a/ovk/core/bundle.py +++ b/ovk/core/bundle.py @@ -22,9 +22,7 @@ def _validate_bundle_inputs(evidence: list[VerificationEvidence]) -> None: subject = evidence[0].subject for index, item in enumerate(evidence[1:], start=1): if item.subject != subject: - raise ValueError( - f"evidence subject mismatch at index {index}: expected {subject}, got {item.subject}" - ) + raise ValueError(f"evidence subject mismatch at index {index}: expected {subject}, got {item.subject}") evidence_ids = [item.evidence_id for item in evidence] duplicate_ids = sorted({item for item in evidence_ids if evidence_ids.count(item) > 1}) if duplicate_ids: diff --git a/ovk/core/compiler_bridge.py b/ovk/core/compiler_bridge.py index f8956a4..5e5cdb0 100644 --- a/ovk/core/compiler_bridge.py +++ b/ovk/core/compiler_bridge.py @@ -33,7 +33,7 @@ from ovk.compilers.infrastructure import compile_kubernetes_objects, compile_terraform_plan from ovk.compilers.infrastructure.ir import InfrastructureIR from ovk.core.execution_models import AbstractionCoverage, MaterialReference -from ovk.core.bundle import content_digest +from ovk.core.materials import material_reference_from_payload def coverage_policy_from_dict(policy: dict[str, Any] | None) -> CoveragePolicy: @@ -351,13 +351,11 @@ def material_refs_from_digest( source_revision: str | None, trusted: bool = False, ) -> MaterialReference: - digest = content_digest(payload) - return MaterialReference( + return material_reference_from_payload( material_id=material_id[:32], - kind=kind, # type: ignore[arg-type] + kind=kind, uri=uri, - sha256=digest, - size_bytes=len(digest), + payload=payload, source_revision=source_revision, trusted=trusted, ) @@ -400,8 +398,7 @@ def register_cbmc_project(data: dict[str, Any]) -> tuple[CbmcProject, Abstractio confidence=0.4, extracted_elements=extracted, expected_elements=None, - warnings=list(project.warnings) - + ["CBMC materials registered without project-grounded strict eligibility"], + warnings=list(project.warnings) + ["CBMC materials registered without project-grounded strict eligibility"], ) compiler_id = "ovk.cbmc.harness_or_cdb.v1" else: diff --git a/ovk/core/context.py b/ovk/core/context.py index 4bb9179..fee1fa2 100644 --- a/ovk/core/context.py +++ b/ovk/core/context.py @@ -66,10 +66,7 @@ def _validate_policy_mapping(loaded: object, *, source: str) -> dict[str, Any]: for error in errors: location = "/".join(str(part) for part in error.path) or "$" formatted.append(f"{location}: {error.message}") - raise ValueError( - f"OVK verification policy from {source} failed schema validation: " - + "; ".join(formatted) - ) + raise ValueError(f"OVK verification policy from {source} failed schema validation: " + "; ".join(formatted)) return dict(loaded) @@ -170,9 +167,7 @@ def budget_from_policy(policy: dict[str, Any]) -> VerificationBudget: denied = normalize_denied_backends(denied_raw) allowed_set = frozenset(allowed) if allowed is not None else None denied_set = frozenset(denied) - max_wall = float( - budget_section.get("max_wall_time_seconds", policy.get("max_wall_time_seconds", 30.0)) - ) + max_wall = float(budget_section.get("max_wall_time_seconds", policy.get("max_wall_time_seconds", 30.0))) max_memory = int(budget_section.get("max_memory_mb", policy.get("max_memory_mb", 512))) routing_section = policy.get("routing", {}) prefer_deterministic = False diff --git a/ovk/core/counterexample_translator.py b/ovk/core/counterexample_translator.py index 739d3da..1abefab 100644 --- a/ovk/core/counterexample_translator.py +++ b/ovk/core/counterexample_translator.py @@ -211,17 +211,14 @@ def _under(root: Path) -> bool: return False if not (_under(cwd) or _under(verification_root) or _under(temp_root)): - raise ValueError( - f"refusing to write generated tests outside workspace or temp: {resolved}" - ) + raise ValueError(f"refusing to write generated tests outside workspace or temp: {resolved}") resolved.mkdir(parents=True, exist_ok=True) written: list[Path] = [] for index, artifact in enumerate(generate_regression_artifacts(bundle)): - failure_mode = "".join( - ch if ch.isalnum() or ch in {"_", "-"} else "_" - for ch in str(artifact["failure_mode"]) - )[:80] + failure_mode = "".join(ch if ch.isalnum() or ch in {"_", "-"} else "_" for ch in str(artifact["failure_mode"]))[ + :80 + ] json_path = resolved / f"regression_{index}_{failure_mode}.json" # Ensure no path traversal via failure_mode. if json_path.resolve().parent != resolved: diff --git a/ovk/core/decision.py b/ovk/core/decision.py index 5633c5e..ba8512d 100644 --- a/ovk/core/decision.py +++ b/ovk/core/decision.py @@ -7,11 +7,7 @@ def evidence_has_status(bundle: EvidenceBundle, status: VerificationStatus) -> bool: """Return true if any backend claim in the bundle has the given status.""" - return any( - claim.status == status - for evidence in bundle.evidence - for claim in evidence.backend_claims - ) + return any(claim.status == status for evidence in bundle.evidence for claim in evidence.backend_claims) def evidence_has_unknown_like(bundle: EvidenceBundle) -> bool: @@ -21,11 +17,7 @@ def evidence_has_unknown_like(bundle: EvidenceBundle) -> bool: VerificationStatus.ERROR, VerificationStatus.SKIPPED, } - return any( - claim.status in unknown_like - for evidence in bundle.evidence - for claim in evidence.backend_claims - ) + return any(claim.status in unknown_like for evidence in bundle.evidence for claim in evidence.backend_claims) def _unknown_like_recommendation( @@ -90,11 +82,16 @@ def decide_with_reason( ) -> dict[str, str]: """Return merge recommendation and human-readable reason for bundle construction.""" recommendation = decide(bundle, enforce=enforce, default_on_unknown=default_on_unknown) - from_unknown = recommendation in { - MergeRecommendation.BLOCK, - MergeRecommendation.REQUIRE_HUMAN_REVIEW, - MergeRecommendation.ALLOW_WITH_WARNING, - } and evidence_has_unknown_like(bundle) and not evidence_has_status(bundle, VerificationStatus.FAIL) + from_unknown = ( + recommendation + in { + MergeRecommendation.BLOCK, + MergeRecommendation.REQUIRE_HUMAN_REVIEW, + MergeRecommendation.ALLOW_WITH_WARNING, + } + and evidence_has_unknown_like(bundle) + and not evidence_has_status(bundle, VerificationStatus.FAIL) + ) return { "merge_recommendation": recommendation.value, "reason": _decision_reason(recommendation, from_unknown=from_unknown), diff --git a/ovk/core/deterministic_evaluators.py b/ovk/core/deterministic_evaluators.py new file mode 100644 index 0000000..9c19d71 --- /dev/null +++ b/ovk/core/deterministic_evaluators.py @@ -0,0 +1,224 @@ +"""Pure deterministic evaluator functions executed in isolated workers.""" + +from __future__ import annotations + +from typing import Any + +from ovk.adapters.ci_secrets.exposure import find_ci_secrets_counterexamples +from ovk.adapters.deployment.state_machine import find_skipped_approval_paths +from ovk.adapters.infra.exposure import find_exposure_counterexamples +from ovk.adapters.infra.validation import validate_infra_input +from ovk.adapters.opa.self_protection import ( + find_self_protection_unknowns, + find_self_protection_violations, +) +from ovk.adapters.z3.counterexample import counterexamples_from_obligation +from ovk.adapters.z3.executor import run_authorization_obligation_with_z3 +from ovk.adapters.z3.obligation import build_authorization_obligation +from ovk.adapters.z3.validation import validate_authorization_input + +EVALUATOR_IDS = frozenset( + { + "authorization-deterministic", + "self-protection-deterministic", + "infrastructure-deterministic", + "ci-secrets-deterministic", + "deployment-deterministic", + "z3-authorization-native", + } +) + + +def evaluate_deterministic(evaluator_id: str, payload: dict[str, Any]) -> dict[str, Any]: + """Run one registered evaluator and return a JSON-serializable result envelope.""" + if evaluator_id not in EVALUATOR_IDS: + return { + "termination": "tool_error", + "exit_code": 1, + "raw_result": { + "status": "error", + "reason": f"unknown evaluator_id: {evaluator_id}", + }, + } + dispatch = { + "authorization-deterministic": _evaluate_authorization_deterministic, + "self-protection-deterministic": _evaluate_self_protection_deterministic, + "infrastructure-deterministic": _evaluate_infrastructure_deterministic, + "ci-secrets-deterministic": _evaluate_ci_secrets_deterministic, + "deployment-deterministic": _evaluate_deployment_deterministic, + "z3-authorization-native": _evaluate_z3_authorization_native, + } + return dispatch[evaluator_id](payload) + + +def _evaluate_authorization_deterministic(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + issues = validate_authorization_input(data) + if issues: + return { + "termination": "invalid_output", + "exit_code": 1, + "raw_result": { + "status": "unknown", + "reason": "malformed authorization input", + "issues": issues, + "models": [], + "counterexamples": [], + }, + } + auth_obligation = build_authorization_obligation(data) + counterexamples = counterexamples_from_obligation(auth_obligation) + return { + "termination": "completed", + "exit_code": 0, + "raw_result": { + "status": "fail" if counterexamples else "pass", + "reason": ( + "deterministic violation witness found" + if counterexamples + else "no deterministic violation witness found" + ), + "models": counterexamples, + "counterexamples": counterexamples, + }, + } + + +def _evaluate_self_protection_deterministic(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + violations = find_self_protection_violations(data) + unknowns = find_self_protection_unknowns(data) + if violations: + status = "fail" + counterexamples = [item.as_counterexample() for item in violations] + elif unknowns: + status = "unknown" + counterexamples = [item.as_counterexample() for item in unknowns] + else: + status = "pass" + counterexamples = [] + return { + "termination": "completed", + "exit_code": 0, + "raw_result": {"status": status, "counterexamples": counterexamples}, + } + + +def _evaluate_infrastructure_deterministic(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + issues = validate_infra_input(data) + if issues: + return { + "termination": "invalid_output", + "exit_code": 1, + "raw_result": { + "status": "unknown", + "counterexamples": [ + { + "summary": issue.message, + "failure_mode": "infrastructure_abstraction_invalid", + "path": issue.path, + } + for issue in issues + ], + }, + } + counterexamples = find_exposure_counterexamples(data) + return { + "termination": "completed", + "exit_code": 0, + "raw_result": { + "status": "fail" if counterexamples else "pass", + "counterexamples": counterexamples, + }, + } + + +def _evaluate_ci_secrets_deterministic(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + workflows = data.get("workflows") + if not isinstance(workflows, list) or not workflows: + return { + "termination": "invalid_output", + "exit_code": 1, + "raw_result": { + "status": "unknown", + "counterexamples": [ + { + "summary": "Workflow abstraction is missing or empty.", + "failure_mode": "missing_workflow_abstraction", + } + ], + }, + } + counterexamples = find_ci_secrets_counterexamples(data) + return { + "termination": "completed", + "exit_code": 0, + "raw_result": { + "status": "fail" if counterexamples else "pass", + "counterexamples": counterexamples, + }, + } + + +def _evaluate_deployment_deterministic(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + if not data.get("states") or not data.get("transitions"): + return { + "termination": "invalid_output", + "exit_code": 1, + "raw_result": { + "status": "unknown", + "counterexamples": [ + { + "summary": "State machine abstraction is missing states or transitions.", + "failure_mode": "missing_state_machine_abstraction", + } + ], + }, + } + counterexamples = find_skipped_approval_paths(data) + return { + "termination": "completed", + "exit_code": 0, + "raw_result": { + "status": "fail" if counterexamples else "pass", + "counterexamples": counterexamples, + }, + } + + +def _evaluate_z3_authorization_native(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + issues = validate_authorization_input(data) + if issues: + return { + "termination": "invalid_output", + "exit_code": 1, + "native_execution": False, + "raw_result": { + "status": "unknown", + "reason": "malformed input", + "issues": issues, + "models": [], + }, + } + auth_obligation = build_authorization_obligation(data) + native_raw = run_authorization_obligation_with_z3(auth_obligation) + from ovk.adapters.z3.result import normalize_z3_authorization_result + + normalized = normalize_z3_authorization_result(native_raw) + native_execution = native_raw.get("reason") != "z3-solver is not installed" + termination = "tool_unavailable" if native_raw.get("reason") == "z3-solver is not installed" else "completed" + return { + "termination": termination, + "exit_code": 0 if termination == "completed" else 1, + "native_execution": native_execution, + "raw_result": { + "status": normalized["status"], + "reason": native_raw.get("reason"), + "models": native_raw.get("models", []), + "counterexamples": normalized.get("counterexamples", []), + }, + } diff --git a/ovk/core/diff_iac.py b/ovk/core/diff_iac.py index c9a8447..7e26d74 100644 --- a/ovk/core/diff_iac.py +++ b/ovk/core/diff_iac.py @@ -17,7 +17,9 @@ def _is_iac_path(path: str) -> bool: suffix = PurePosixPath(normalized).suffix if suffix in IAC_SUFFIXES: return True - return any(marker in normalized for marker in ("/k8s/", "/kubernetes/", "/deploy/", "deployment")) or normalized.startswith(("k8s/", "kubernetes/")) + return any( + marker in normalized for marker in ("/k8s/", "/kubernetes/", "/deploy/", "deployment") + ) or normalized.startswith(("k8s/", "kubernetes/")) def _iam_policy_is_overly_permissive(resource_type: str, lines: list[str]) -> bool: @@ -27,9 +29,9 @@ def _iam_policy_is_overly_permissive(resource_type: str, lines: list[str]) -> bo joined = "\n".join(lines).lower() if "iam_policy" not in resource_type.lower(): return False - if 'principal' in joined and "*" in joined and ("admin" in joined or "action" in joined): + if "principal" in joined and "*" in joined and ("admin" in joined or "action" in joined): return True - return 'action' in joined and 'resource' in joined and "*" in joined + return "action" in joined and "resource" in joined and "*" in joined def _parse_tf_hunk(content: str) -> dict | None: diff --git a/ovk/core/doctor.py b/ovk/core/doctor.py index 02ee907..962f9c5 100644 --- a/ovk/core/doctor.py +++ b/ovk/core/doctor.py @@ -122,7 +122,9 @@ def _check_manifest_example() -> DoctorCheck: try: data = read_json_file(manifest) lanes = data.get("lanes", []) - return DoctorCheck("example_manifest", isinstance(lanes, list) and len(lanes) >= 5, f"{len(lanes)} lanes in example manifest") + return DoctorCheck( + "example_manifest", isinstance(lanes, list) and len(lanes) >= 5, f"{len(lanes)} lanes in example manifest" + ) except Exception as error: # noqa: BLE001 return DoctorCheck("example_manifest", False, str(error)) diff --git a/ovk/core/evidence_from_execution.py b/ovk/core/evidence_from_execution.py index b204b7d..3dc5fe0 100644 --- a/ovk/core/evidence_from_execution.py +++ b/ovk/core/evidence_from_execution.py @@ -2,11 +2,18 @@ from __future__ import annotations + from typing import Any + from ovk.compilers.authorization import CoveragePolicy, strict_allow_permitted + from ovk.core.bundle import content_digest + from ovk.core.execution_models import ObligationExecutionRecord + +from ovk.core.materials import material_set_digest_for_obligation + from ovk.core.models import BackendClaim, MergeRecommendation, VerificationEvidence, VerificationStatus @@ -21,9 +28,13 @@ def execution_record_to_evidence( coverage_policy: CoveragePolicy | None = None, ) -> VerificationEvidence: """Project an obligation execution record into public evidence.""" + obligation = record.obligation + routing = record.routing + counterexamples: list[dict[str, Any]] = [] + for result in record.results: counterexamples.extend(result.counterexamples) @@ -35,16 +46,13 @@ def execution_record_to_evidence( assumptions=list(result.assumptions), limits=list(result.limits), adapter_version=next( - ( - item.adapter_version - for item in record.backend_obligations - if item.backend == result.backend - ), + (item.adapter_version for item in record.backend_obligations if item.backend == result.backend), None, ), ) for result in sorted(record.results, key=lambda item: item.backend) ] + if not claims: claims = [ BackendClaim( @@ -56,11 +64,18 @@ def execution_record_to_evidence( ) ] + material_payloads = [item.model_dump(mode="json") for item in obligation.materials] + + material_set_digest = material_set_digest_for_obligation(obligation) + artifacts: list[dict[str, Any]] = [] + for result in record.results: artifacts.extend(result.generated_artifacts) + for item in record.open_obligations: artifacts.append(dict(item)) + artifacts.append( { "kind": "routing_enforced", @@ -69,6 +84,28 @@ def execution_record_to_evidence( "obligation_id": obligation.obligation_id, } ) + + artifacts.append( + { + "kind": "control_plane_trace", + "compiler": { + "compiler_id": obligation.compiler_id, + "compiler_version": obligation.compiler_version, + }, + "coverage": obligation.coverage.model_dump(mode="json"), + "material_set_digest": material_set_digest, + "routing_id": routing.routing_id, + "requested_backends": list(routing.requested), + "eligible_backends": [item.backend for item in routing.eligible], + "selected_backends": [item.backend for item in routing.selected], + "attempted_backends": [item.backend for item in record.attempts], + "executed_backends": [item.backend for item in record.results], + "execution_attempts": [item.model_dump(mode="json") for item in record.attempts], + "routing_enforced": routing_enforced, + "aggregation_policy": routing.aggregation_policy, + } + ) + if obligation.compiler_id: artifacts.append( { @@ -76,22 +113,27 @@ def execution_record_to_evidence( "compiler_id": obligation.compiler_id, "compiler_version": obligation.compiler_version, "coverage": obligation.coverage.model_dump(mode="json"), - "materials": [item.model_dump(mode="json") for item in obligation.materials], + "materials": material_payloads, + "material_set_digest": material_set_digest, } ) recommendation = record.merge_recommendation + aggregation_reason = record.aggregation_reason - # Incomplete abstraction cannot produce allow under strict coverage policy. + policy = coverage_policy or CoveragePolicy() + allow_ok = obligation.abstraction.get("strict_allow_permitted") + if allow_ok is None: allow_ok = strict_allow_permitted(obligation.coverage, policy) + if recommendation == MergeRecommendation.ALLOW and not allow_ok: recommendation = MergeRecommendation.REQUIRE_HUMAN_REVIEW - aggregation_reason = ( - f"{aggregation_reason}; incomplete abstraction cannot allow under strict coverage" - ) + + aggregation_reason = f"{aggregation_reason}; incomplete abstraction cannot allow under strict coverage" + artifacts.append( { "kind": "incomplete_abstraction", @@ -105,12 +147,16 @@ def execution_record_to_evidence( "human_review_required": recommendation.value != "allow", "aggregation_reason": aggregation_reason, "routing_enforced": routing_enforced, + "fallback_used": record.fallback_used, + "fallback_accepted": record.fallback_accepted, + "fallback_cause": record.fallback_cause, } evidence_id = content_digest( { "obligation_id": obligation.obligation_id, "routing_id": routing.routing_id, + "material_set_digest": material_set_digest, "results": [claim.model_dump(mode="json") for claim in claims], } )[:24] @@ -118,7 +164,7 @@ def execution_record_to_evidence( return VerificationEvidence( evidence_id=f"ev-{evidence_id}", schema_version=schema_version, - subject=obligation.subject.model_dump(mode="json"), + subject={key: value for key, value in obligation.subject.model_dump(mode="json").items() if value is not None}, change_origin={"author_type": author_type, "agent": agent, "task": task}, intent={ "intent_id": obligation.intent_id, @@ -131,11 +177,12 @@ def execution_record_to_evidence( decision=decision, obligation_id=obligation.obligation_id, routing_id=routing.routing_id, + material_set_digest=material_set_digest if schema_version.endswith(".v3") else None, compiler={ "compiler_id": obligation.compiler_id, "compiler_version": obligation.compiler_version, }, - materials=[item.model_dump(mode="json") for item in obligation.materials], + materials=material_payloads, coverage=obligation.coverage.model_dump(mode="json"), requested_backends=list(routing.requested), eligible_backends=[item.backend for item in routing.eligible], diff --git a/ovk/core/evidence_invariants.py b/ovk/core/evidence_invariants.py index 5ab00f7..7eda9bf 100644 --- a/ovk/core/evidence_invariants.py +++ b/ovk/core/evidence_invariants.py @@ -6,6 +6,7 @@ from typing import Any from ovk.core.bundle import content_digest +from ovk.core.materials import compute_material_set_digest from ovk.core.models import EvidenceBundle, VerificationStatus @@ -230,7 +231,8 @@ def _check_control_plane_invariants(bundle: EvidenceBundle) -> list[EvidenceInva for index, evidence in enumerate(bundle.evidence): path = f"evidence[{index}]" is_v2 = str(evidence.schema_version).endswith(".v2") or evidence.routing_enforced - if not is_v2 and evidence.obligation_id is None and evidence.routing_id is None: + is_v3 = str(evidence.schema_version).endswith(".v3") + if not is_v2 and not is_v3 and evidence.obligation_id is None and evidence.routing_id is None: continue selected = list(evidence.selected_backends or []) @@ -310,7 +312,11 @@ def _check_control_plane_invariants(bundle: EvidenceBundle) -> list[EvidenceInva ) ) - if evidence.aggregation_policy and evidence.decision.get("aggregation_reason") is None and evidence.routing_enforced: + if ( + evidence.aggregation_policy + and evidence.decision.get("aggregation_reason") is None + and evidence.routing_enforced + ): # aggregation_reason is optional on legacy decision dicts; required for enforced v2. if "aggregation_reason" not in evidence.decision: issues.append( @@ -332,6 +338,23 @@ def _check_control_plane_invariants(bundle: EvidenceBundle) -> list[EvidenceInva message="material digests must be present (OVK-INV-015)", ) ) + if is_v3 or evidence.routing_enforced: + expected_material_set = compute_material_set_digest(evidence.materials) + stated_material_set = evidence.material_set_digest + if not stated_material_set: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.material_set_digest", + message="evidence v3 must include material_set_digest (OVK-INV-021)", + ) + ) + elif stated_material_set != expected_material_set: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.material_set_digest", + message="material_set_digest must match recomputed canonical digest (OVK-INV-021)", + ) + ) coverage = evidence.coverage or {} recommendation = _decision_value(evidence.decision, "merge_recommendation") @@ -383,7 +406,33 @@ def _check_control_plane_invariants(bundle: EvidenceBundle) -> list[EvidenceInva ) ) - # OVK-INV-020 checked at attestation binding time; also flag missing routing_id on enforced. + if is_v3 or evidence.routing_enforced: + trace_artifacts = [a for a in evidence.generated_artifacts if a.get("kind") == "control_plane_trace"] + if not trace_artifacts and is_v3: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.generated_artifacts", + message="evidence v3 must include control_plane_trace artifact (OVK-INV-022)", + ) + ) + elif trace_artifacts: + trace = trace_artifacts[0] + if trace.get("routing_id") not in {None, evidence.routing_id}: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.generated_artifacts", + message="control_plane_trace routing_id must match evidence routing_id (OVK-INV-022)", + ) + ) + trace_digest = trace.get("material_set_digest") + if is_v3 and trace_digest and trace_digest != evidence.material_set_digest: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.generated_artifacts", + message="control_plane_trace material_set_digest must match evidence (OVK-INV-021)", + ) + ) + if evidence.routing_enforced and not evidence.routing_id: issues.append( EvidenceInvariantIssue( diff --git a/ovk/core/execution_budget.py b/ovk/core/execution_budget.py index aaeeb36..bf921fd 100644 --- a/ovk/core/execution_budget.py +++ b/ovk/core/execution_budget.py @@ -3,8 +3,7 @@ ``ExecutionBudget`` is defined in ``ovk.core.execution_models``. This module re-exports it, provides policy conversion helpers, and defines the worker protocol that enforces subprocess environment bounds. Adapters describe -computation; workers enforce timeout, cwd, an environment allowlist, and -output caps. +computation; workers enforce timeout, cwd, env allowlist, and output caps. """ from __future__ import annotations @@ -126,11 +125,12 @@ def run( @dataclass class LocalSubprocessWorker: - """Local subprocess worker with timeout, cwd bound, and minimal environment. + """Local subprocess worker with timeout, cwd bound, and env allowlist. - Only keys in ``allowed_env_keys`` are inherited from the parent. Callers may - pass explicit non-secret variables through ``env``. Known credential keys - are rejected in both inherited and explicit environments. + Child processes inherit only configured safe parent variables, plus any + explicit non-secret additions supplied by the caller. Secret-bearing + names are always stripped from inherited and explicit environments. + Non-positive wall-time budgets are rejected without starting a process. """ allowed_env_keys: frozenset[str] = field( @@ -227,18 +227,29 @@ def run( ) def _build_env(self, extra: Mapping[str, str] | None) -> dict[str, str]: - """Build a minimal child environment from allowlisted and explicit keys.""" - allowed_upper = {key.upper() for key in self.allowed_env_keys} - baseline = { - key: value - for key, value in os.environ.items() - if key.upper() in allowed_upper and key.upper() not in _SECRET_ENV_DENYLIST - } + """Build a child environment from the configured allowlist only. + + Parent variables are inherited only when their names appear in + ``allowed_env_keys``. Explicit ``extra`` keys are merged unless they + match the secret denylist. Secret-bearing names are always removed. + """ + allow = {key.upper() for key in self.allowed_env_keys} + baseline: dict[str, str] = {} + for key, value in os.environ.items(): + if key.upper() in allow and key.upper() not in _SECRET_ENV_DENYLIST: + baseline[key] = value if extra: for key, value in extra.items(): if key.upper() in _SECRET_ENV_DENYLIST: continue baseline[key] = value + for denied in _SECRET_ENV_DENYLIST: + baseline.pop(denied, None) + baseline.pop(denied.lower(), None) + # Drop any case variant that might have been injected. + for existing in list(baseline): + if existing.upper() == denied.upper(): + baseline.pop(existing, None) return baseline diff --git a/ovk/core/execution_models.py b/ovk/core/execution_models.py index 5170bf4..ea1384b 100644 --- a/ovk/core/execution_models.py +++ b/ovk/core/execution_models.py @@ -214,9 +214,21 @@ class FallbackPolicy(BaseModel): allow_fallback: bool = False fallback_backends: list[str] = Field(default_factory=list) + acceptable_fallback_guarantees: list[str] = Field(default_factory=list) on_timeout: FallbackOutcome = "unknown" on_tool_unavailable: FallbackOutcome = "unknown" on_invalid_output: FallbackOutcome = "unknown" + on_resource_exhausted: FallbackOutcome = "unknown" + + def outcome_for_termination(self, termination: TerminationKind | str) -> FallbackOutcome: + """Map a termination kind to the configured fallback outcome.""" + mapping: dict[str, FallbackOutcome] = { + "timeout": self.on_timeout, + "tool_unavailable": self.on_tool_unavailable, + "invalid_output": self.on_invalid_output, + "resource_exhausted": self.on_resource_exhausted, + } + return mapping.get(str(termination), "unknown") class BackendCapabilityAssessment(BaseModel): @@ -429,6 +441,25 @@ class NormalizedBackendResult(BaseModel): generated_artifacts: list[dict[str, Any]] = Field(default_factory=list) +class CachedBackendExecution(BaseModel): + """Provenance-preserving cache payload for a backend execution (ovk.cache.v3). + + Cache hits must replay this record rather than synthesizing a new attempt or + re-inferring ``native_execution`` from current tool availability. + """ + + schema_version: Literal["ovk.cache.v3"] = "ovk.cache.v3" + attempt: ExecutionAttempt + native_execution: bool + tool_version: str | None = None + tool_digest: str | None = None + termination: TerminationKind + exit_code: int | None = None + raw_result_digest: str | None = None + environment_fingerprint: str + normalized_result: NormalizedBackendResult + + class ObligationExecutionRecord(BaseModel): """Full typed record of compiling, routing, and executing one obligation.""" @@ -441,6 +472,9 @@ class ObligationExecutionRecord(BaseModel): merge_recommendation: MergeRecommendation aggregation_reason: str open_obligations: list[dict[str, Any]] = Field(default_factory=list) + fallback_used: bool = False + fallback_accepted: bool = False + fallback_cause: str | None = None # --------------------------------------------------------------------------- @@ -557,15 +591,18 @@ def compute_backend_obligation_id( def attempt_digest_input(attempt: ExecutionAttempt | dict[str, Any]) -> dict[str, Any]: - """Canonical digest payload for an execution attempt (excludes ``attempt_id`` and wall-clock times). + """Canonical digest payload for an execution attempt (excludes identity and timing). - Timestamps are excluded so identical executions remain content-addressable when - wall-clock values differ; ``duration_ms`` and termination/output digests remain. + ``attempt_id``, wall-clock timestamps, and ``duration_ms`` are excluded so + otherwise equivalent executions remain content-addressable across sequential, + parallel, cached, and uncached runs. Duration remains observational metadata + on the attempt object itself. """ data = dict(_dump_json(attempt)) data.pop("attempt_id", None) data.pop("started_at", None) data.pop("finished_at", None) + data.pop("duration_ms", None) return data diff --git a/ovk/core/infrastructure_compiler.py b/ovk/core/infrastructure_compiler.py index a78c1b4..19e801d 100644 --- a/ovk/core/infrastructure_compiler.py +++ b/ovk/core/infrastructure_compiler.py @@ -75,8 +75,7 @@ def compile_infrastructure_obligation( coverage=coverage, acceptable_guarantees=["exposure_graph_check"], required_capabilities=["infrastructure"], - policy_digest=policy_digest - or content_digest({"lane": "infrastructure", "policy": policy or {}}), + policy_digest=policy_digest or content_digest({"lane": "infrastructure", "policy": policy or {}}), ) return provisional.model_copy(update={"obligation_id": compute_obligation_id(provisional)}) diff --git a/ovk/core/kernel.py b/ovk/core/kernel.py index 0d89915..b4201dc 100644 --- a/ovk/core/kernel.py +++ b/ovk/core/kernel.py @@ -22,6 +22,7 @@ from ovk.core.surface_routing import surface_backend_bonuses from ovk.core.result_cache import DEFAULT_CACHE_DIR from ovk.core.risk_ranker import rank_intents +from ovk.core.routing_pipeline import AuthoritativeRoutingPlan, build_authoritative_routing_plan from ovk.core.router import VerificationBudget, route_intent from ovk.paths import resource_path @@ -108,14 +109,6 @@ def execute_kernel( base_sha=base_sha, ) - routing, routing_by_intent = _routing_for_plan( - plan, - context=ctx, - budget=budget, - template_dir=template_dir or resource_path("templates"), - adapter_dir=adapter_dir or resource_path("adapters"), - ) - compiler = ObligationCompilerRegistry.default() obligations = compiler.compile( plan, @@ -127,6 +120,31 @@ def execute_kernel( ) decision_options = bundle_decision_options(ctx.policy) + policy_dict = ctx.policy if isinstance(ctx.policy, dict) else None + + authoritative_plan: AuthoritativeRoutingPlan | None = None + routing_by_intent: dict[str, Any] = {} + routing: list[dict[str, Any]] = [] + + if obligations: + authoritative_plan = build_authoritative_routing_plan( + obligations, + policy=policy_dict, + repo=ctx.repo, + head_sha=ctx.head_sha, + base_sha=ctx.base_sha, + ) + routing_by_intent = authoritative_plan.legacy_routing_by_intent() + routing = authoritative_plan.routing_metadata_list() + else: + routing, routing_by_intent = _routing_for_plan( + plan, + context=ctx, + budget=budget, + template_dir=template_dir or resource_path("templates"), + adapter_dir=adapter_dir or resource_path("adapters"), + ) + if not obligations: bundle = make_bundle( [ @@ -150,7 +168,8 @@ def execute_kernel( cache_dir=cache_dir, use_cache=use_cache, parallel=parallel, - policy=ctx.policy if isinstance(ctx.policy, dict) else None, + policy=policy_dict, + evidence_schema_version="ovk.evidence.v3", ) bundle = make_bundle(evidence_items, **decision_options) diff --git a/ovk/core/lane_compiler.py b/ovk/core/lane_compiler.py index c8e0e9b..c1a4214 100644 --- a/ovk/core/lane_compiler.py +++ b/ovk/core/lane_compiler.py @@ -60,10 +60,14 @@ def compile_lane_inputs_from_plan( if diff_text and is_unified_diff(diff_text): if "ci_secrets" in lanes_needed: for index, data in enumerate(workflow_inputs_from_diff(diff_text)): - jobs.append({"lane": "ci_secrets", "data": data, "input_format": "infra", "job_id": f"ci_secrets_{index}"}) + jobs.append( + {"lane": "ci_secrets", "data": data, "input_format": "infra", "job_id": f"ci_secrets_{index}"} + ) if "authorization" in lanes_needed: for index, data in enumerate(authorization_inputs_from_diff(diff_text)): - jobs.append({"lane": "authorization", "data": data, "input_format": "infra", "job_id": f"authorization_{index}"}) + jobs.append( + {"lane": "authorization", "data": data, "input_format": "infra", "job_id": f"authorization_{index}"} + ) if "infrastructure" in lanes_needed: for index, item in enumerate(infra_inputs_from_diff(diff_text)): jobs.append( @@ -76,7 +80,9 @@ def compile_lane_inputs_from_plan( ) if "deployment" in lanes_needed: for index, data in enumerate(deployment_inputs_from_diff(diff_text)): - jobs.append({"lane": "deployment", "data": data, "input_format": "infra", "job_id": f"deployment_{index}"}) + jobs.append( + {"lane": "deployment", "data": data, "input_format": "infra", "job_id": f"deployment_{index}"} + ) if "backend" in lanes_needed: candidate_intents = set(plan.get("candidate_intents", [])) for index, data in enumerate(cbmc_inputs_from_diff(diff_text)): @@ -97,7 +103,9 @@ def compile_lane_inputs_from_plan( if isinstance(suggested.get("ci_secrets"), list): existing = sum(1 for job in jobs if job["lane"] == "ci_secrets") for index, data in enumerate(suggested["ci_secrets"][existing:]): - jobs.append({"lane": "ci_secrets", "data": data, "input_format": "infra", "job_id": f"ci_secrets_suggested_{index}"}) + jobs.append( + {"lane": "ci_secrets", "data": data, "input_format": "infra", "job_id": f"ci_secrets_suggested_{index}"} + ) return jobs diff --git a/ovk/core/materials.py b/ovk/core/materials.py index 4223e71..3e84ddc 100644 --- a/ovk/core/materials.py +++ b/ovk/core/materials.py @@ -6,7 +6,7 @@ from typing import Any, cast from ovk.core.bundle import content_digest -from ovk.core.execution_models import MaterialKind, MaterialReference +from ovk.core.execution_models import MaterialKind, MaterialReference, VerificationObligation def canonical_material_bytes(payload: Any) -> bytes: @@ -19,6 +19,29 @@ def canonical_material_bytes(payload: Any) -> bytes: ).encode("utf-8") +def compute_material_set_digest(materials: list[Any] | None) -> str: + """Compute a canonical digest over sorted material ids and content digests.""" + entries: list[dict[str, str]] = [] + for item in materials or []: + if hasattr(item, "model_dump"): + payload = item.model_dump(mode="json") + elif isinstance(item, dict): + payload = item + else: + continue + material_id = str(payload.get("material_id") or payload.get("id") or "") + digest = str(payload.get("sha256") or payload.get("digest") or "") + if material_id or digest: + entries.append({"material_id": material_id, "sha256": digest}) + entries.sort(key=lambda row: (row["material_id"], row["sha256"])) + return content_digest({"materials": entries}) + + +def material_set_digest_for_obligation(obligation: VerificationObligation) -> str: + """Return the canonical material-set digest for one typed obligation.""" + return compute_material_set_digest([item.model_dump(mode="json") for item in obligation.materials]) + + def material_reference_from_payload( *, material_id: str, diff --git a/ovk/core/models.py b/ovk/core/models.py index 4f0e346..d0940cb 100644 --- a/ovk/core/models.py +++ b/ovk/core/models.py @@ -96,6 +96,7 @@ class VerificationEvidence(BaseModel): routing_id: str | None = None compiler: dict[str, Any] | None = None materials: list[dict[str, Any]] | None = None + material_set_digest: str | None = None coverage: dict[str, Any] | None = None requested_backends: list[str] | None = None eligible_backends: list[str] | None = None diff --git a/ovk/core/multi_lane.py b/ovk/core/multi_lane.py index f058fed..e0331f8 100644 --- a/ovk/core/multi_lane.py +++ b/ovk/core/multi_lane.py @@ -39,8 +39,7 @@ def _validate_lane_input(data: dict[str, Any], schema_name: str, *, lane: str) - report = validate_against_schema(data, schema) if not report.valid: issues = "; ".join( - f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" - for issue in report.issues + f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" for issue in report.issues ) raise ValueError(f"{lane} input failed schema validation: {issues}") @@ -138,8 +137,7 @@ def load_verification_manifest(path: Path, *, validate: bool = True) -> dict[str report = validate_against_schema(manifest, schema) if not report.valid: issues = "; ".join( - f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" - for issue in report.issues + f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" for issue in report.issues ) raise ValueError(f"verification manifest failed schema validation: {issues}") return manifest @@ -200,9 +198,7 @@ def _evaluate_manifest_entry( return None input_path = _resolve_manifest_file(manifest_root, str(input_value), field="input") policy_path = ( - _resolve_manifest_file(manifest_root, str(entry["policy"]), field="policy") - if entry.get("policy") - else None + _resolve_manifest_file(manifest_root, str(entry["policy"]), field="policy") if entry.get("policy") else None ) data = read_json_file(input_path) evidence = evaluate_lane( @@ -220,9 +216,7 @@ def _evaluate_manifest_entry( "input_format": str(entry.get("input_format", "infra")), "policy": entry.get("policy"), } - return evidence.model_copy( - update={"evidence_id": f"{evidence.evidence_id}-{content_digest(identity)[:12]}"} - ) + return evidence.model_copy(update={"evidence_id": f"{evidence.evidence_id}-{content_digest(identity)[:12]}"}) def run_verification_manifest( diff --git a/ovk/core/native_backend_probe.py b/ovk/core/native_backend_probe.py index 6eeb06a..032fe6d 100644 --- a/ovk/core/native_backend_probe.py +++ b/ovk/core/native_backend_probe.py @@ -218,7 +218,9 @@ def probe_all_native_backends() -> list[NativeBackendSummary]: continue binary_present = any(item.binary_present for item in results) fixture_matches = all(item.runtime_status == item.oracle_status for item in results) - native_used = all(item.used_native_binary for item in results if item.binary_present) if binary_present else False + native_used = ( + all(item.used_native_binary for item in results if item.binary_present) if binary_present else False + ) if backend in TIER1_NATIVE_EXECUTION_BACKENDS and binary_present: native_used = all(item.used_native_binary for item in results) summaries.append( diff --git a/ovk/core/obligation_compiler.py b/ovk/core/obligation_compiler.py index 005d199..c054bc2 100644 --- a/ovk/core/obligation_compiler.py +++ b/ovk/core/obligation_compiler.py @@ -48,10 +48,13 @@ def compile( obligations: list[dict[str, Any]] = [] for job in jobs: lane = str(job["lane"]) - intent_id = str(job.get("intent_id") or next( - (intent for intent, mapped in self._intent_to_lane.items() if mapped == lane), - lane, - )) + intent_id = str( + job.get("intent_id") + or next( + (intent for intent, mapped in self._intent_to_lane.items() if mapped == lane), + lane, + ) + ) obligations.append( { "intent_id": intent_id, diff --git a/ovk/core/output_validation.py b/ovk/core/output_validation.py index d258a64..3f9a561 100644 --- a/ovk/core/output_validation.py +++ b/ovk/core/output_validation.py @@ -69,8 +69,7 @@ def missing_release_layout_schema_coverage(layout: dict[str, Any]) -> list[str]: def _issues_from_pydantic(error: ValidationError) -> list[ValidationIssue]: return [ - ValidationIssue(path=[str(part) for part in issue["loc"]], message=issue["msg"]) - for issue in error.errors() + ValidationIssue(path=[str(part) for part in issue["loc"]], message=issue["msg"]) for issue in error.errors() ] @@ -113,10 +112,7 @@ def validate_generated_file(path: Path, kind: str) -> ValidationReport: def _format_file_validation_failures(path: Path, report: ValidationReport) -> list[str]: - return [ - f"{path.name} validation at {issue.path}: {issue.message}" - for issue in report.issues - ] + return [f"{path.name} validation at {issue.path}: {issue.message}" for issue in report.issues] def validate_output_directory(root: Path) -> list[str]: diff --git a/ovk/core/planner.py b/ovk/core/planner.py index 7231598..e6d6040 100644 --- a/ovk/core/planner.py +++ b/ovk/core/planner.py @@ -16,9 +16,33 @@ from ovk.core.diff_parser import extract_changed_paths, is_unified_diff from ovk.core.intent_registry import IntentRegistry from ovk.core.router import route_intent +from ovk.core.routing_pipeline import build_authoritative_routing_plan from ovk.paths import resource_path +def plan_from_lane_obligations( + obligations: list[dict[str, Any]], + *, + policy: dict[str, Any] | None = None, + repo: str = "unknown/repo", + head_sha: str = "unknown", + base_sha: str | None = None, +) -> dict[str, Any]: + """Build a plan whose routing uses compile-then-route authoritative decisions.""" + plan = build_authoritative_routing_plan( + obligations, + policy=policy, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + ) + return { + "obligations": obligations, + "routing": plan.routing_metadata_list(), + "routing_by_intent": plan.legacy_routing_by_intent(), + } + + def plan_from_changed_files( changed_files: list[str], *, diff --git a/ovk/core/policy_config.py b/ovk/core/policy_config.py index 4ffa5b2..33d7f34 100644 --- a/ovk/core/policy_config.py +++ b/ovk/core/policy_config.py @@ -6,9 +6,7 @@ from ovk.core.router import RoutingConfig, routing_config_from_policy -VALID_DEFAULT_ON_UNKNOWN = frozenset( - {"require_human_review", "block", "allow_with_warning"} -) +VALID_DEFAULT_ON_UNKNOWN = frozenset({"require_human_review", "block", "allow_with_warning"}) def resolve_default_on_unknown(policy: dict[str, Any] | None) -> str: diff --git a/ovk/core/provenance.py b/ovk/core/provenance.py index cc59bdc..20deb1d 100644 --- a/ovk/core/provenance.py +++ b/ovk/core/provenance.py @@ -87,14 +87,13 @@ def build_provenance_statement( "control_plane": { "obligation_ids": [item.obligation_id for item in bundle.evidence if item.obligation_id], "routing_ids": [item.routing_id for item in bundle.evidence if item.routing_id], + "material_set_digests": [item.material_set_digest for item in bundle.evidence if item.material_set_digest], "routing_enforced": [bool(item.routing_enforced) for item in bundle.evidence], "compilers": [item.compiler for item in bundle.evidence if item.compiler], "coverage": [item.coverage for item in bundle.evidence if item.coverage], "selected_backends": [item.selected_backends for item in bundle.evidence], "executed_backends": [item.executed_backends for item in bundle.evidence], - "guarantees": [ - [claim.guarantee_type for claim in item.backend_claims] for item in bundle.evidence - ], + "guarantees": [[claim.guarantee_type for claim in item.backend_claims] for item in bundle.evidence], }, "materials": [material_entry(path, workspace=workspace) for path in materials or []], "invocation": invocation diff --git a/ovk/core/release_preflight_report.py b/ovk/core/release_preflight_report.py index 84b079b..2546e1d 100644 --- a/ovk/core/release_preflight_report.py +++ b/ovk/core/release_preflight_report.py @@ -121,9 +121,7 @@ def _check_smoke_quality_reports() -> list[str]: make_bundle( [ evaluate_ci_secrets_exposure( - read_json_file( - resource_path("examples", "ci_secrets", "input_secrets_safe.json") - ), + read_json_file(resource_path("examples", "ci_secrets", "input_secrets_safe.json")), repo="smoke/repo", head_sha="smoke-head", ) @@ -189,13 +187,9 @@ def _check_pilot_program() -> list[str]: failures: list[str] = [] for result in report["results"]: if not result["passed"]: - failures.append( - f"pilot manifest {result['name']} did not allow: {result['merge_recommendation']}" - ) + failures.append(f"pilot manifest {result['name']} did not allow: {result['merge_recommendation']}") if report["manifests_passed"] != report["manifests_total"]: - failures.append( - f"pilot program passed {report['manifests_passed']}/{report['manifests_total']} manifests" - ) + failures.append(f"pilot program passed {report['manifests_passed']}/{report['manifests_total']} manifests") return failures @@ -271,9 +265,7 @@ def build_release_preflight_report() -> PreflightReport: check_from_failures("release_layout_schema_coverage", _check_release_layout_schema_coverage()), check_from_failures("adapter_capabilities", _check_adapter_capabilities()), ), - optional_checks=( - check_from_failures("pilot_metrics_dry_run", _check_pilot_metrics_dry_run()), - ), + optional_checks=(check_from_failures("pilot_metrics_dry_run", _check_pilot_metrics_dry_run()),), ) diff --git a/ovk/core/render.py b/ovk/core/render.py index 521c451..a754a61 100644 --- a/ovk/core/render.py +++ b/ovk/core/render.py @@ -12,7 +12,8 @@ def _section_kind(evidence: VerificationEvidence) -> str: artifacts = evidence.generated_artifacts or [] kinds = {str(item.get("kind")) for item in artifacts if isinstance(item, dict)} if "incomplete_abstraction" in kinds or ( - isinstance(evidence.coverage, dict) and evidence.coverage.get("status") in {"partial", "unknown"} + isinstance(evidence.coverage, dict) + and evidence.coverage.get("status") in {"partial", "unknown"} and recommendation == "require_human_review" and "incomplete" in str(evidence.decision.get("aggregation_reason", "")) ): @@ -127,15 +128,14 @@ def render_evidence_markdown(evidence: VerificationEvidence) -> str: if hint.get("line_hunk") is not None: location += f", line {hint['line_hunk']}" location += ")" - lines.append( - f"- `{hint['fix_class']}`: {hint['suggested_action']}{location}" - ) + lines.append(f"- `{hint['fix_class']}`: {hint['suggested_action']}{location}") open_items = [ item for item in (evidence.generated_artifacts or []) if isinstance(item, dict) - and item.get("kind") in {"backend_disagreement", "quality_error", "incomplete_abstraction", "aggregation_warning"} + and item.get("kind") + in {"backend_disagreement", "quality_error", "incomplete_abstraction", "aggregation_warning"} ] if open_items: lines.extend(["", "Open obligations:"]) diff --git a/ovk/core/result_cache.py b/ovk/core/result_cache.py index 9cadd5d..574887f 100644 --- a/ovk/core/result_cache.py +++ b/ovk/core/result_cache.py @@ -24,12 +24,14 @@ from ovk.core.execution_models import ( BackendEnvironmentFingerprint, BackendObligation, + CachedBackendExecution, NormalizedBackendResult, RoutingDecision, VerificationObligation, ) -CACHE_SCHEMA_VERSION = "ovk.cache.v2" +CACHE_SCHEMA_VERSION = "ovk.cache.v3" +CACHE_SCHEMA_VERSION_V2 = "ovk.cache.v2" DEFAULT_CACHE_DIR = Path(".verification/cache") DEFAULT_TTL_SECONDS = 86400 @@ -262,18 +264,37 @@ def get(self, components: dict[str, Any]) -> CacheEntry | None: def put_backend_result( self, components: dict[str, Any], - result: NormalizedBackendResult, + result: CachedBackendExecution, *, meta: dict[str, Any] | None = None, ) -> str: + if not isinstance(result, CachedBackendExecution): + raise TypeError("ovk.cache.v3 requires CachedBackendExecution; v2 result-only entries are invalid") return self.put(components, result.model_dump(mode="json"), meta=meta) def get_backend_result(self, components: dict[str, Any]) -> NormalizedBackendResult | None: + cached = self.get_cached_execution(components) + return None if cached is None else cached.normalized_result + + def get_cached_execution(self, components: dict[str, Any]) -> CachedBackendExecution | None: + """Return a provenance-preserving v3 cache hit, or None. + + Explicitly invalidates v2 (result-only) entries. + """ entry = self.get(components) if entry is None: return None + payload = entry.payload + schema = payload.get("schema_version") + if schema == CACHE_SCHEMA_VERSION_V2 or schema != CACHE_SCHEMA_VERSION: + # Invalidate non-v3 entries so provenance cannot be reconstructed incorrectly. + namespace = str(components.get("namespace") or NAMESPACE_BACKEND_RESULTS) + key_digest = digest_key_components(components) + path = self.namespace_dir(namespace) / f"{key_digest}.json" + path.unlink(missing_ok=True) + return None try: - return NormalizedBackendResult.model_validate(entry.payload) + return CachedBackendExecution.model_validate(payload) except Exception: # noqa: BLE001 - corrupt cache is a miss return None @@ -366,13 +387,13 @@ def __init__(self, hardened: HardenedResultCache | None = None) -> None: def bind_components(self, key: str, components: dict[str, Any]) -> None: self._last_components[key] = components - def get(self, key: str) -> NormalizedBackendResult | None: + def get(self, key: str) -> CachedBackendExecution | None: components = self._last_components.get(key) if components is None: return None - return self._cache.get_backend_result(components) + return self._cache.get_cached_execution(components) - def put(self, key: str, value: NormalizedBackendResult, *, meta: dict[str, Any]) -> None: + def put(self, key: str, value: CachedBackendExecution, *, meta: dict[str, Any]) -> None: components = self._last_components.get(key) if components is None: return diff --git a/ovk/core/router.py b/ovk/core/router.py index 2b259e6..067ec4f 100644 --- a/ovk/core/router.py +++ b/ovk/core/router.py @@ -110,7 +110,9 @@ def routing_config_from_policy(policy: Mapping[str, Any] | None) -> RoutingConfi else "primary_with_optional_corroboration" ) enforced_raw = section.get("enforced_lanes") or policy.get("enforced_lanes") or [] - enforced = frozenset(str(item) for item in enforced_raw) if isinstance(enforced_raw, (list, tuple, set)) else frozenset() + enforced = ( + frozenset(str(item) for item in enforced_raw) if isinstance(enforced_raw, (list, tuple, set)) else frozenset() + ) return RoutingConfig( mode=mode, strategy=strategy, @@ -237,6 +239,32 @@ def select_primary_with_optional_corroboration( ) ) continue + if not assessment.coverage_requirements_met: + score = _adjust_score_for_preferences( + backend=assessment.backend, + score=float(assessment.score), + prefer_deterministic=config.prefer_deterministic, + ) + if config.accept_partial_primary: + eligible.append( + BackendCandidate( + backend=assessment.backend, + score=score, + support="partial", + guarantee_type=assessment.guarantee_type, + reasons=reasons + ["incomplete coverage; not eligible as required primary"], + native_available=assessment.native_available, + ) + ) + else: + rejected.append( + BackendRejection( + backend=assessment.backend, + reason="coverage requirements not met", + support=assessment.support, + ) + ) + continue if not _guarantee_acceptable(assessment.guarantee_type, acceptable_guarantees): rejected.append( BackendRejection( @@ -289,7 +317,11 @@ def select_primary_with_optional_corroboration( eligible_sorted = sorted(eligible, key=lambda item: (-item.score, item.backend)) primary_candidates = [item for item in eligible_sorted if item.support == "supported"] if not primary_candidates and config.accept_partial_primary: - primary_candidates = list(eligible_sorted) + primary_candidates = [ + item + for item in eligible_sorted + if item.support == "partial" and not any("incomplete coverage" in reason for reason in item.reasons) + ] if primary_candidates: primary = primary_candidates[0] @@ -447,13 +479,7 @@ def _manifest_assessment( support: Literal["supported", "partial", "unsupported", "unavailable"] = "supported" # Explicit relevance from domain/kind match replaces the former constant 1.0 term. relevance = 1.0 if kind_match else 0.5 - score = ( - relevance - + guarantee_strength - + (0.15 * historical_success) - + surface_bonus - - cost - ) + score = relevance + guarantee_strength + (0.15 * historical_success) + surface_bonus - cost reasons.append(f"supports domain {domain} and property kind {property_kind}") elif domain_match: support = "partial" @@ -482,13 +508,16 @@ def _manifest_assessment( prefer_deterministic=budget.prefer_deterministic, ) + material_requirements_met = domain_match and kind_match + coverage_requirements_met = support == "supported" and material_requirements_met + return BackendCapabilityAssessment( backend=tool_name, support=support, score=score, guarantee_type=guarantee, - material_requirements_met=True, - coverage_requirements_met=True, + material_requirements_met=material_requirements_met, + coverage_requirements_met=coverage_requirements_met, native_available=False, estimated_wall_time_seconds=cost * 30, estimated_memory_mb=256, diff --git a/ovk/core/routing_pipeline.py b/ovk/core/routing_pipeline.py new file mode 100644 index 0000000..12d0376 --- /dev/null +++ b/ovk/core/routing_pipeline.py @@ -0,0 +1,340 @@ +"""Single authoritative routing pipeline for typed obligations. + +Compiles backend-neutral obligations before routing, routes each obligation +exactly once via ``route_obligation``, and exposes the same ``routing_id`` to +kernel, CLI, MCP, and evidence emitters. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Mapping + +from ovk.adapters.authorization import build_authorization_registry +from ovk.adapters.ci_secrets import build_ci_secrets_registry +from ovk.adapters.deployment import build_deployment_registry +from ovk.adapters.infrastructure import build_infrastructure_registry +from ovk.adapters.self_protection import build_self_protection_registry +from ovk.core.authorization_compiler import compile_authorization_obligation +from ovk.core.backend_registry import BackendRegistry +from ovk.core.ci_secrets_compiler import compile_ci_secrets_obligation +from ovk.core.deployment_compiler import compile_deployment_obligation +from ovk.core.execution_budget import execution_budget_from_policy +from ovk.core.execution_models import ExecutionContext, RoutingDecision, VerificationObligation +from ovk.core.infrastructure_compiler import compile_infrastructure_obligation +from ovk.core.policy_config import routing_enforced_for_lane +from ovk.core.router import RoutingConfig, route_obligation, routing_config_from_policy, routing_decision_to_legacy_dict +from ovk.core.self_protection_compiler import compile_self_protection_obligation, resolve_metadata_trusted + +LANE_TO_INTENT = { + "self_protection": "agent-cannot-disable-own-ci-gate", + "authorization": "no-admin-route-bypass", + "infrastructure": "no-public-sensitive-resource", + "ci_secrets": "no-secrets-in-untrusted-context", + "deployment": "no-skipped-approval-state", +} + + +def intent_id_for_obligation(obligation: dict[str, Any]) -> str: + """Resolve the canonical intent id for a lane obligation dict.""" + lane = str(obligation["lane"]) + return str(obligation.get("intent_id") or LANE_TO_INTENT.get(lane, lane)) + + +RegistryBuilder = Callable[[], BackendRegistry] +CompilerFn = Callable[..., VerificationObligation] + +_LANE_REGISTRY: dict[str, RegistryBuilder] = { + "authorization": build_authorization_registry, + "self_protection": build_self_protection_registry, + "infrastructure": build_infrastructure_registry, + "ci_secrets": build_ci_secrets_registry, + "deployment": build_deployment_registry, +} + + +def _compile_authorization( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, + policy: dict[str, Any] | None, +) -> VerificationObligation: + return compile_authorization_obligation( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + + +def _compile_self_protection( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, + policy: dict[str, Any] | None, +) -> VerificationObligation: + return compile_self_protection_obligation( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + metadata_trusted=resolve_metadata_trusted(policy), + ) + + +def _compile_infrastructure( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, + policy: dict[str, Any] | None, +) -> VerificationObligation: + return compile_infrastructure_obligation( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + + +def _compile_ci_secrets( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, + policy: dict[str, Any] | None, +) -> VerificationObligation: + return compile_ci_secrets_obligation( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + + +def _compile_deployment( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, + policy: dict[str, Any] | None, +) -> VerificationObligation: + return compile_deployment_obligation( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + + +_LANE_COMPILERS: dict[str, CompilerFn] = { + "authorization": _compile_authorization, + "self_protection": _compile_self_protection, + "infrastructure": _compile_infrastructure, + "ci_secrets": _compile_ci_secrets, + "deployment": _compile_deployment, +} + + +@dataclass(frozen=True) +class AuthoritativeRoutingPlan: + """Typed obligations and immutable routing decisions keyed by intent_id.""" + + typed_obligations: dict[str, VerificationObligation] + routing_by_intent: dict[str, RoutingDecision] + + def routing_metadata_list(self) -> list[dict[str, Any]]: + return [ + routing_decision_to_legacy_dict(decision, intent_id=intent_id) + for intent_id, decision in sorted(self.routing_by_intent.items()) + ] + + def legacy_routing_by_intent(self) -> dict[str, dict[str, Any]]: + return { + intent_id: routing_decision_to_legacy_dict(decision, intent_id=intent_id) + for intent_id, decision in self.routing_by_intent.items() + } + + +def compile_typed_obligation( + *, + lane: str, + data: dict[str, Any], + repo: str, + head_sha: str, + base_sha: str | None = None, + policy: dict[str, Any] | None = None, +) -> VerificationObligation: + """Compile one lane input into a backend-neutral obligation.""" + compiler = _LANE_COMPILERS.get(lane) + if compiler is None: + raise ValueError(f"no typed compiler registered for lane {lane!r}") + return compiler( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + + +def route_compiled_obligation( + obligation: VerificationObligation, + *, + lane: str, + policy: dict[str, Any] | None = None, +) -> RoutingDecision: + """Route a compiled obligation exactly once through ``route_obligation``.""" + registry_builder = _LANE_REGISTRY.get(lane) + if registry_builder is None: + raise ValueError(f"no registry registered for lane {lane!r}") + registry = registry_builder() + routing_config = routing_config_from_policy(policy) + budget = execution_budget_from_policy(policy) + enforced = routing_enforced_for_lane(policy, lane) + context = ExecutionContext( + subject=obligation.subject, + budget=budget, + policy_digest=obligation.policy_digest, + metadata={"enforced": enforced, "lane": lane}, + ) + config = RoutingConfig( + mode="enforced" if enforced else routing_config.mode, + strategy=routing_config.strategy, + aggregation=routing_config.aggregation, + max_selected_backends=routing_config.max_selected_backends, + prefer_deterministic=routing_config.prefer_deterministic, + allow_fallback=routing_config.allow_fallback, + accept_partial_primary=routing_config.accept_partial_primary, + enforced_lanes=frozenset({lane}) if enforced else routing_config.enforced_lanes, + ) + return route_obligation( + obligation, + registry, + context=context, + config=config, + policy=policy, + ) + + +def build_authoritative_routing_plan( + obligations: list[dict[str, Any]], + *, + policy: dict[str, Any] | None = None, + repo: str, + head_sha: str, + base_sha: str | None = None, +) -> AuthoritativeRoutingPlan: + """Compile obligations before routing; route each obligation exactly once. + + Experimental lanes without a typed compiler (for example ``backend``/CBMC) + are skipped here rather than crashing the production control plane. Those + obligations remain available to legacy evaluators when present. + """ + typed: dict[str, VerificationObligation] = {} + routing: dict[str, RoutingDecision] = {} + for item in obligations: + lane = str(item["lane"]) + if lane not in _LANE_COMPILERS or lane not in _LANE_REGISTRY: + # Fail closed for unknown *production* claims elsewhere; skip + # catalog/experimental lanes that lack typed compilers. + continue + intent_id = intent_id_for_obligation(item) + data = dict(item["input"]) + obligation = compile_typed_obligation( + lane=lane, + data=data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + decision = route_compiled_obligation(obligation, lane=lane, policy=policy) + typed[intent_id] = obligation + routing[intent_id] = decision + return AuthoritativeRoutingPlan(typed_obligations=typed, routing_by_intent=routing) + + +def coerce_routing_decision( + routing: RoutingDecision | Mapping[str, Any] | None, + *, + intent_id: str, +) -> RoutingDecision | None: + """Return a typed routing decision when available.""" + if routing is None: + return None + if isinstance(routing, RoutingDecision): + return routing + try: + return RoutingDecision.model_validate(dict(routing)) + except Exception: + routing_id = routing.get("routing_id") + if not routing_id: + return None + return RoutingDecision.model_validate( + { + **routing, + "obligation_id": routing.get("obligation_id") or intent_id, + } + ) + + +def ensure_authoritative_routing( + obligations: list[dict[str, Any]], + routing_by_intent: Mapping[str, RoutingDecision | Mapping[str, Any] | None] | None, + *, + policy: dict[str, Any] | None, + repo: str, + head_sha: str, + base_sha: str | None = None, +) -> AuthoritativeRoutingPlan: + """Merge caller routing with compile-then-route defaults for all obligations.""" + plan = build_authoritative_routing_plan( + obligations, + policy=policy, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + ) + if not routing_by_intent: + return plan + + merged_routing = dict(plan.routing_by_intent) + merged_typed = dict(plan.typed_obligations) + for obligation in obligations: + intent_id = intent_id_for_obligation(obligation) + provided = routing_by_intent.get(intent_id) + coerced = coerce_routing_decision(provided, intent_id=intent_id) + if coerced is not None: + merged_routing[intent_id] = coerced + return AuthoritativeRoutingPlan( + typed_obligations=merged_typed, + routing_by_intent=merged_routing, + ) + + +def require_routing_decision( + routing: RoutingDecision | Mapping[str, Any] | None, + *, + intent_id: str, + lane: str, + policy: dict[str, Any] | None, +) -> RoutingDecision: + """Fail closed when an enforced lane lacks a pre-computed routing decision.""" + decision = coerce_routing_decision(routing, intent_id=intent_id) + if decision is not None: + return decision + if routing_enforced_for_lane(policy, lane): + raise RuntimeError(f"enforced lane {lane!r} requires authoritative RoutingDecision for intent {intent_id!r}") + raise RuntimeError(f"missing routing decision for intent {intent_id!r}") diff --git a/ovk/core/schema_validation.py b/ovk/core/schema_validation.py index 8ce7e80..00f87f2 100644 --- a/ovk/core/schema_validation.py +++ b/ovk/core/schema_validation.py @@ -72,7 +72,6 @@ def require_schema_valid( if report.valid: return issues = "; ".join( - f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" - for issue in report.issues + f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" for issue in report.issues ) raise ValueError(f"{context} failed schema validation: {issues}") diff --git a/ovk/core/self_protection_compiler.py b/ovk/core/self_protection_compiler.py index ac1b480..86dea3e 100644 --- a/ovk/core/self_protection_compiler.py +++ b/ovk/core/self_protection_compiler.py @@ -17,6 +17,29 @@ COMPILER_ID = "ovk.self_protection.neutral.v1" COMPILER_VERSION = "0.1.0" +TRUSTED_METADATA_PROVENANCE_KINDS: frozenset[str] = frozenset( + { + "protected_base_workflow", + "signed_service", + "maintainer_supplied", + } +) + + +def resolve_metadata_trusted(policy: dict[str, Any] | None) -> bool: + """Return True only when policy supplies explicit trusted provenance.""" + if not isinstance(policy, dict): + return False + trust = policy.get("trust") + if not isinstance(trust, dict): + return False + if not bool(trust.get("metadata_trusted")): + return False + provenance = trust.get("provenance_kind") or trust.get("provenance") + if provenance not in TRUSTED_METADATA_PROVENANCE_KINDS: + return False + return True + def _phase(data: dict[str, Any], name: str) -> dict[str, Any]: value = data.get(name, {}) @@ -30,12 +53,13 @@ def compile_self_protection_obligation( head_sha: str, base_sha: str | None = None, policy_digest: str | None = None, - metadata_trusted: bool = True, + metadata_trusted: bool = False, ) -> VerificationObligation: """Compile a self-protection obligation with base/head metadata materials. When ``metadata_trusted`` is True, before/after required-check materials are marked trusted. Untrusted metadata cannot authorize allow under enforcement. + Trust requires explicit policy provenance via ``resolve_metadata_trusted``. """ before = _phase(data, "before") after = _phase(data, "after") diff --git a/ovk/core/shadow_obligation.py b/ovk/core/shadow_obligation.py index 4eeaefc..50f1099 100644 --- a/ovk/core/shadow_obligation.py +++ b/ovk/core/shadow_obligation.py @@ -7,11 +7,11 @@ from ovk.core.bundle import content_digest from ovk.core.execution_models import ( AbstractionCoverage, - MaterialReference, VerificationObligation, compute_abstraction_digest, compute_obligation_id, ) +from ovk.core.materials import material_reference_from_payload from ovk.core.models import RiskSeverity, VerificationSubject LANE_PROPERTY_KIND: dict[str, str] = { @@ -43,12 +43,11 @@ def build_shadow_obligation( ) -> VerificationObligation: """Construct a backend-neutral obligation for shadow control-plane execution.""" subject = VerificationSubject(repo=repo, head_sha=head_sha, base_sha=base_sha) - material = MaterialReference( + material = material_reference_from_payload( material_id=content_digest({"lane": lane, "input": data})[:32], kind="diff", uri=f"ovk-material:lane/{lane}", - sha256=content_digest(data), - size_bytes=len(content_digest(data)), + payload=data, source_revision=head_sha, trusted=False, ) diff --git a/ovk/core/sigstore_signing.py b/ovk/core/sigstore_signing.py index 63465ab..cb9f48b 100644 --- a/ovk/core/sigstore_signing.py +++ b/ovk/core/sigstore_signing.py @@ -39,9 +39,7 @@ def _configured_trust_identity() -> tuple[str, str]: identity = os.environ.get(COSIGN_IDENTITY_ENV, "").strip() issuer = os.environ.get(COSIGN_ISSUER_ENV, "").strip() if not identity or not issuer: - raise RuntimeError( - "OVK Sigstore signing requires OVK_COSIGN_IDENTITY and OVK_COSIGN_ISSUER" - ) + raise RuntimeError("OVK Sigstore signing requires OVK_COSIGN_IDENTITY and OVK_COSIGN_ISSUER") return identity, issuer diff --git a/ovk/core/source_profile_evidence.py b/ovk/core/source_profile_evidence.py new file mode 100644 index 0000000..1001fdf --- /dev/null +++ b/ovk/core/source_profile_evidence.py @@ -0,0 +1,273 @@ +"""Execute bounded source-profile proofs for template conformance v2. + +Statuses such as ``source_profile_strict_eligible`` must derive from executed +semantic evidence (compiler runs on fixtures), not mere file presence. +``externally_calibrated_strict`` is never granted by local generation alone. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from ovk.compilers.authorization.fastapi_ast import FastApiAstAuthorizationCompiler +from ovk.compilers.authorization.material_loader import materials_from_pair +from ovk.compilers.github_actions.permissions import extract_permissions, has_write_token +from ovk.compilers.github_actions.secrets import extract_secrets +from ovk.compilers.github_actions.trust_flow import compile_workflow_trust +from ovk.compilers.infrastructure.kubernetes import compile_kubernetes_objects +from ovk.compilers.infrastructure.terraform_plan import compile_terraform_plan +from ovk.core.source_profiles import ( + KNOWN_SOURCE_PROFILES, + is_known_source_profile, + source_profile_strict_eligible, +) + + +@dataclass(frozen=True) +class ProfileSemanticEvidence: + profile_id: str + materials_trusted: bool + coverage_complete: bool + enforcement_test_present: bool + notes: tuple[str, ...] = () + + def as_dict(self) -> dict[str, Any]: + return { + "profile_id": self.profile_id, + "materials_trusted": self.materials_trusted, + "coverage_complete": self.coverage_complete, + "enforcement_test_present": self.enforcement_test_present, + "strict_eligible": source_profile_strict_eligible( + profile_id=self.profile_id, + materials_trusted=self.materials_trusted, + coverage_complete=self.coverage_complete, + enforcement_test_present=self.enforcement_test_present, + ), + "notes": list(self.notes), + } + + +def _enforcement_present(repo_root: Path, relative: str | None) -> bool: + return bool(relative) and (repo_root / relative).is_file() + + +def prove_fastapi_ast_profile(repo_root: Path, *, enforcement_test: str | None) -> ProfileSemanticEvidence: + profile_id = "authorization.fastapi.ast_v1" + base = ( + "from fastapi import Depends, FastAPI\n" + "def require_admin():\n" + " return 'admin'\n" + "app = FastAPI()\n" + "@app.get('/admin/users', dependencies=[Depends(require_admin)])\n" + "def users():\n" + " return []\n" + ) + materials = materials_from_pair(path="app.py", base_source=base, head_source=base) + ir = FastApiAstAuthorizationCompiler().compile(materials) + profile_ok = any(profile_id in note for note in ir.warnings) + coverage_complete = bool(ir.routes) and all(route.support == "supported" for route in ir.routes) + notes = [ + f"routes={len(ir.routes)}", + f"profile_marker={'yes' if profile_ok else 'no'}", + ] + return ProfileSemanticEvidence( + profile_id=profile_id, + materials_trusted=True, # fixture materials authored for this proof + coverage_complete=coverage_complete and profile_ok and not ir.unsupported_constructs, + enforcement_test_present=_enforcement_present(repo_root, enforcement_test), + notes=tuple(notes), + ) + + +def prove_terraform_recursive_profile(repo_root: Path, *, enforcement_test: str | None) -> ProfileSemanticEvidence: + profile_id = "infrastructure.terraform.plan_recursive_v1" + plan = { + "format_version": "1.2", + "planned_values": { + "root_module": { + "resources": [], + "child_modules": [ + { + "address": "module.exports", + "resources": [ + { + "address": "module.exports.aws_s3_bucket.data", + "type": "aws_s3_bucket", + "name": "data", + "values": { + "tags": {"sensitivity": "confidential"}, + "acl": "public-read", + }, + } + ], + "child_modules": [], + } + ], + } + }, + } + ir = compile_terraform_plan(plan) + profile_ok = any(profile_id in note for note in ir.warnings) + coverage_complete = bool(ir.resources) and profile_ok + return ProfileSemanticEvidence( + profile_id=profile_id, + materials_trusted=True, + coverage_complete=coverage_complete, + enforcement_test_present=_enforcement_present(repo_root, enforcement_test), + notes=(f"resources={len(ir.resources)}", f"eligibility={ir.eligibility}"), + ) + + +def prove_k8s_controller_profile(repo_root: Path, *, enforcement_test: str | None) -> ProfileSemanticEvidence: + profile_id = "infrastructure.kubernetes.controller_reachability_v1" + objects = [ + { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": "api", "namespace": "default"}, + "spec": {"type": "LoadBalancer", "selector": {"app": "api"}}, + }, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "api", + "namespace": "default", + "annotations": {"ovk.io/sensitivity": "confidential"}, + }, + "spec": { + "template": { + "metadata": {"labels": {"app": "api"}}, + "spec": {"containers": [{"name": "api", "image": "api:1"}]}, + } + }, + }, + ] + ir = compile_kubernetes_objects(objects) + profile_ok = any(profile_id in note for note in ir.warnings) + has_selector_edge = any(edge.kind == "service_selector" for edge in ir.edges) + return ProfileSemanticEvidence( + profile_id=profile_id, + materials_trusted=True, + coverage_complete=profile_ok and has_selector_edge, + enforcement_test_present=_enforcement_present(repo_root, enforcement_test), + notes=(f"edges={len(ir.edges)}", f"selector_edge={'yes' if has_selector_edge else 'no'}"), + ) + + +def prove_actions_permissions_flow(repo_root: Path, *, enforcement_test: str | None) -> ProfileSemanticEvidence: + profile_id = "ci_secrets.actions.permissions_flow_v1" + workflow = { + "_ovk_path": "ci.yml", + "on": {"pull_request_target": {}}, + "permissions": {"contents": "write"}, + "jobs": { + "build": { + "runs-on": "ubuntu-latest", + "steps": [ + { + "name": "use secret", + "run": "echo ${{ secrets.DEPLOY_TOKEN }}", + "env": {"TOKEN": "${{ secrets.DEPLOY_TOKEN }}"}, + } + ], + } + }, + } + ir = compile_workflow_trust(workflow) + grants = extract_permissions(workflow) + secrets = extract_secrets(workflow) + write = has_write_token(grants) + findings = list(ir.findings) if hasattr(ir, "findings") else [] + coverage_complete = bool(secrets) and write and (bool(findings) or bool(ir.edges)) + return ProfileSemanticEvidence( + profile_id=profile_id, + materials_trusted=True, + coverage_complete=coverage_complete, + enforcement_test_present=_enforcement_present(repo_root, enforcement_test), + notes=( + f"secrets={len(secrets)}", + f"write_token={write}", + f"findings={len(findings)}", + f"compiled_with_source_profile:{profile_id}", + ), + ) + + +def prove_deployment_trusted_profile(repo_root: Path, *, enforcement_test: str | None) -> ProfileSemanticEvidence: + """Deployment strictness only when an explicit trusted profile marker is present.""" + profile_id = "deployment.trusted_profile_v1" + # Without an explicit trusted profile material, coverage is incomplete by design. + trusted_marker = repo_root / "examples" / "deployment_state" / "trusted_profile.v1.json" + materials_trusted = trusted_marker.is_file() + coverage_complete = False + notes = ["requires explicit trusted_profile material for strictness"] + if materials_trusted: + try: + from ovk.core.json_io import read_json_file + + payload = read_json_file(trusted_marker) + coverage_complete = ( + isinstance(payload, dict) + and payload.get("source_profile") == profile_id + and payload.get("trusted") is True + ) + notes = [f"trusted_marker={trusted_marker.as_posix()}"] + except (OSError, ValueError) as exc: + notes = [f"trusted_marker_unreadable:{exc}"] + materials_trusted = False + return ProfileSemanticEvidence( + profile_id=profile_id, + materials_trusted=materials_trusted, + coverage_complete=coverage_complete, + enforcement_test_present=_enforcement_present(repo_root, enforcement_test), + notes=tuple(notes), + ) + + +_PROVERS = { + "authorization.fastapi.ast_v1": prove_fastapi_ast_profile, + "infrastructure.terraform.plan_recursive_v1": prove_terraform_recursive_profile, + "infrastructure.kubernetes.controller_reachability_v1": prove_k8s_controller_profile, + "ci_secrets.actions.permissions_flow_v1": prove_actions_permissions_flow, + "deployment.trusted_profile_v1": prove_deployment_trusted_profile, +} + + +def collect_source_profile_evidence( + repo_root: Path, + *, + catalog_by_intent: dict[str, dict[str, Any]], +) -> dict[str, ProfileSemanticEvidence]: + """Run profile proofs for intents that declare a source_profile_id.""" + evidence: dict[str, ProfileSemanticEvidence] = {} + for intent_id, entry in catalog_by_intent.items(): + profile_id = entry.get("source_profile_id") + if not is_known_source_profile(str(profile_id) if profile_id else None): + continue + prover = _PROVERS.get(str(profile_id)) + if prover is None: + evidence[intent_id] = ProfileSemanticEvidence( + profile_id=str(profile_id), + materials_trusted=False, + coverage_complete=False, + enforcement_test_present=False, + notes=("no prover registered for profile",), + ) + continue + links = entry.get("links") or {} + evidence[intent_id] = prover( + repo_root, + enforcement_test=str(links.get("enforcement_test") or "") or None, + ) + return evidence + + +def evidence_payload(evidence: dict[str, ProfileSemanticEvidence]) -> dict[str, Any]: + return { + "schema_version": "ovk.source_profile_evidence.v1", + "known_profiles": sorted(KNOWN_SOURCE_PROFILES), + "intents": {intent: item.as_dict() for intent, item in sorted(evidence.items())}, + } diff --git a/ovk/core/source_profiles.py b/ovk/core/source_profiles.py new file mode 100644 index 0000000..3486c84 --- /dev/null +++ b/ovk/core/source_profiles.py @@ -0,0 +1,93 @@ +"""Source-profile identifiers and eligibility gates (Sprint 6). + +Profiles authorize deeper compilers when trusted materials are present. +They do not by themselves grant ``externally_calibrated_strict`` status. +""" + +from __future__ import annotations + +from typing import Any, Literal + +SourceProfileId = Literal[ + "authorization.fastapi.ast_v1", + "authorization.express.ast_v1", + "infrastructure.terraform.plan_recursive_v1", + "infrastructure.kubernetes.controller_reachability_v1", + "ci_secrets.actions.permissions_flow_v1", + "deployment.trusted_profile_v1", +] + +KNOWN_SOURCE_PROFILES: frozenset[str] = frozenset( + { + "authorization.fastapi.ast_v1", + "authorization.express.ast_v1", + "infrastructure.terraform.plan_recursive_v1", + "infrastructure.kubernetes.controller_reachability_v1", + "ci_secrets.actions.permissions_flow_v1", + "deployment.trusted_profile_v1", + } +) + +# Maps profile IDs to the production compiler entrypoints that implement them. +PROFILE_COMPILER_BINDINGS: dict[str, str] = { + "authorization.fastapi.ast_v1": "ovk.compilers.authorization.fastapi_ast:FastApiAstAuthorizationCompiler", + "authorization.express.ast_v1": "ovk.compilers.authorization.express:ExpressAuthorizationCompiler", + "infrastructure.terraform.plan_recursive_v1": "ovk.compilers.infrastructure.terraform_plan:compile_terraform_plan", + "infrastructure.kubernetes.controller_reachability_v1": "ovk.compilers.infrastructure.kubernetes:compile_kubernetes_objects", + "ci_secrets.actions.permissions_flow_v1": "ovk.compilers.github_actions.trust_flow:compile_workflow_trust", + "deployment.trusted_profile_v1": "ovk.compilers.deployment.explicit_schema:compile_explicit_schema", +} + +LANE_DEFAULT_PROFILES: dict[str, tuple[str, ...]] = { + "authorization": ( + "authorization.fastapi.ast_v1", + "authorization.express.ast_v1", + ), + "infrastructure": ( + "infrastructure.terraform.plan_recursive_v1", + "infrastructure.kubernetes.controller_reachability_v1", + ), + "ci_secrets": ("ci_secrets.actions.permissions_flow_v1",), + "deployment": ("deployment.trusted_profile_v1",), +} + +# Remaining gaps (honest): Express module-graph depth, Actions composite recursion +# beyond current trust_flow expansion, and deployment strictness without an +# explicit trusted_profile material. + + +def is_known_source_profile(profile_id: str | None) -> bool: + return bool(profile_id) and profile_id in KNOWN_SOURCE_PROFILES + + +def source_profile_strict_eligible( + *, + profile_id: str | None, + materials_trusted: bool, + coverage_complete: bool, + enforcement_test_present: bool, +) -> bool: + """Return True when a template may claim source_profile_strict_eligible.""" + return is_known_source_profile(profile_id) and materials_trusted and coverage_complete and enforcement_test_present + + +def profiles_from_policy(policy: dict[str, Any] | None, *, lane: str) -> list[str]: + """Extract requested source profiles for a lane from repository policy.""" + if not isinstance(policy, dict): + return [] + section = policy.get("source_profiles") + if not isinstance(section, dict): + return [] + raw = section.get(lane, section.get("profiles", [])) + if isinstance(raw, str): + raw = [raw] + if not isinstance(raw, list): + return [] + return [str(item) for item in raw if is_known_source_profile(str(item))] + + +def compiler_binding_for(profile_id: str) -> str | None: + """Return the compiler binding string for a known profile, if any.""" + if not is_known_source_profile(profile_id): + return None + return PROFILE_COMPILER_BINDINGS.get(profile_id) diff --git a/ovk/core/template_conformance.py b/ovk/core/template_conformance.py index a9c7bfe..17e17f2 100644 --- a/ovk/core/template_conformance.py +++ b/ovk/core/template_conformance.py @@ -3,6 +3,9 @@ A template is ``catalog_only`` unless every required executable link exists. Public executable claims that lack a compiler/adapter/example/test path are downgraded honestly rather than advertised as strict-eligible. + +PR10 / Sprint 7 semantic statuses (``conformance_status_v2``) derive from +executed source-profile evidence, not mere file presence. """ from __future__ import annotations @@ -14,6 +17,11 @@ from typing import Any, Literal from ovk.core.json_io import read_json_file +from ovk.core.source_profile_evidence import ( + ProfileSemanticEvidence, + collect_source_profile_evidence, +) +from ovk.core.source_profiles import source_profile_strict_eligible ProductionStatus = Literal[ "catalog_only", @@ -23,6 +31,14 @@ "deprecated", ] +ProductionStatusV2 = Literal[ + "catalog_only", + "executable_advisory", + "source_profile_strict_eligible", + "externally_calibrated_strict", + "deprecated", +] + REQUIRED_ROW_FIELDS = ( "intent_id", "path", @@ -54,6 +70,7 @@ EXECUTABLE_CATALOG: dict[str, dict[str, Any]] = { "no-admin-route-bypass": { "lane": "authorization", + "source_profile_id": "authorization.fastapi.ast_v1", "claimed_backends": ["z3-native", "authorization-deterministic"], "links": { "lane_evaluator": "ovk/adapters/authorization/deterministic_adapter.py", @@ -67,6 +84,8 @@ }, "agent-cannot-disable-own-ci-gate": { "lane": "self_protection", + # No source-profile prover yet; stays executable_advisory under v2. + "source_profile_id": None, "claimed_backends": ["opa-native", "self-protection-deterministic"], "links": { "lane_evaluator": "ovk/adapters/self_protection/deterministic_adapter.py", @@ -80,6 +99,7 @@ }, "no-public-sensitive-resource": { "lane": "infrastructure", + "source_profile_id": "infrastructure.terraform.plan_recursive_v1", "claimed_backends": ["infrastructure-deterministic"], "links": { "lane_evaluator": "ovk/adapters/infrastructure/deterministic_adapter.py", @@ -93,6 +113,7 @@ }, "no-secrets-in-untrusted-context": { "lane": "ci_secrets", + "source_profile_id": "ci_secrets.actions.permissions_flow_v1", "claimed_backends": ["ci-secrets-deterministic"], "links": { "lane_evaluator": "ovk/adapters/ci_secrets/deterministic_adapter.py", @@ -106,6 +127,7 @@ }, "no-skipped-approval-state": { "lane": "deployment", + "source_profile_id": "deployment.trusted_profile_v1", "claimed_backends": ["deployment-deterministic"], "links": { "lane_evaluator": "ovk/adapters/deployment/deterministic_adapter.py", @@ -143,6 +165,14 @@ "strict_eligible": 4, } +STATUS_RANK_V2 = { + "deprecated": 0, + "catalog_only": 1, + "executable_advisory": 2, + "source_profile_strict_eligible": 3, + "externally_calibrated_strict": 4, +} + @dataclass(frozen=True) class TemplateConformanceRow: @@ -159,14 +189,49 @@ class TemplateConformanceRow: missing_executable_links: list[str] lane: str | None notes: list[str] + source_profile_id: str | None = None + profile_evidence: ProfileSemanticEvidence | None = None + externally_calibrated: bool = False + + def semantic_status_v2(self) -> ProductionStatusV2: + """Derive PR10 semantic status from executed evidence, not file presence alone.""" + if self.production_status == "deprecated": + return "deprecated" + if self.externally_calibrated and self.profile_evidence is not None: + evidence = self.profile_evidence + if source_profile_strict_eligible( + profile_id=evidence.profile_id, + materials_trusted=evidence.materials_trusted, + coverage_complete=evidence.coverage_complete, + enforcement_test_present=evidence.enforcement_test_present, + ): + return "externally_calibrated_strict" + if self.profile_evidence is not None: + evidence = self.profile_evidence + if source_profile_strict_eligible( + profile_id=evidence.profile_id, + materials_trusted=evidence.materials_trusted, + coverage_complete=evidence.coverage_complete, + enforcement_test_present=evidence.enforcement_test_present, + ): + return "source_profile_strict_eligible" + if ( + self.production_status in {"strict_eligible", "advisory", "experimental"} + and not self.missing_executable_links + ): + return "executable_advisory" + if self.production_status in {"advisory", "experimental"}: + return "executable_advisory" + return "catalog_only" def to_dict(self) -> dict[str, Any]: - return { + payload = { "intent_id": self.intent_id, "path": self.path, "domain": self.domain, "version": self.version, "production_status": self.production_status, + "conformance_status_v2": self.semantic_status_v2(), "risk_severity": self.risk_severity, "property_kind": self.property_kind, "acceptable_evidence_kinds": list(self.acceptable_evidence_kinds), @@ -176,6 +241,11 @@ def to_dict(self) -> dict[str, Any]: "lane": self.lane, "notes": list(self.notes), } + if self.source_profile_id: + payload["source_profile_id"] = self.source_profile_id + if self.profile_evidence is not None: + payload["source_profile_evidence"] = self.profile_evidence.as_dict() + return payload def _infer_claimed_backends(intent_id: str, title: str, catalog_entry: dict[str, Any] | None) -> list[str]: @@ -211,6 +281,7 @@ def classify_template( repo_root: Path, intent_path: Path, template: dict[str, Any], + profile_evidence: ProfileSemanticEvidence | None = None, ) -> TemplateConformanceRow: """Classify one template into an honest production status.""" intent_id = str(template.get("intent_id") or intent_path.stem) @@ -224,6 +295,9 @@ def classify_template( ) notes: list[str] = [] lane = str(catalog_entry["lane"]) if catalog_entry else None + source_profile_id = None + if catalog_entry and catalog_entry.get("source_profile_id"): + source_profile_id = str(catalog_entry["source_profile_id"]) if not missing: status: ProductionStatus = str(catalog_entry.get("max_status", "strict_eligible")) # type: ignore[assignment] @@ -241,25 +315,24 @@ def classify_template( # Honest downgrade for native-named templates without executable catalog entry. if claimed and intent_id not in EXECUTABLE_CATALOG: status = _min_status(status, "catalog_only") - notes.append( - "downgraded unsupported public executable claim for backends: " + ", ".join(claimed) - ) + notes.append("downgraded unsupported public executable claim for backends: " + ", ".join(claimed)) if template.get("deprecated") is True: status = "deprecated" notes.append("template marked deprecated") + if profile_evidence is not None: + notes.append( + "source_profile_evidence:" + + ("strict_ok" if profile_evidence.as_dict()["strict_eligible"] else "incomplete") + ) + relative = intent_path.relative_to(repo_root).as_posix() if intent_path.is_absolute() else intent_path.as_posix() evidence = template.get("acceptable_evidence") or [] - evidence_kinds = sorted( - { - str(item.get("kind")) - for item in evidence - if isinstance(item, dict) and item.get("kind") - } - ) + evidence_kinds = sorted({str(item.get("kind")) for item in evidence if isinstance(item, dict) and item.get("kind")}) risk = template.get("risk") if isinstance(template.get("risk"), dict) else {} prop = template.get("property") if isinstance(template.get("property"), dict) else {} + externally_calibrated = bool(template.get("externally_calibrated") is True) return TemplateConformanceRow( intent_id=intent_id, path=relative, @@ -274,18 +347,34 @@ def classify_template( missing_executable_links=missing, lane=lane, notes=notes, + source_profile_id=source_profile_id, + profile_evidence=profile_evidence, + externally_calibrated=externally_calibrated, ) def build_conformance_matrix(repo_root: Path, templates_dir: Path | None = None) -> dict[str, Any]: """Scan templates and build the conformance matrix document.""" templates_dir = templates_dir or (repo_root / "templates") + profile_evidence = collect_source_profile_evidence( + repo_root, + catalog_by_intent=EXECUTABLE_CATALOG, + ) rows: list[TemplateConformanceRow] = [] for path in sorted(templates_dir.rglob("*.intent.json")): template = read_json_file(path) - rows.append(classify_template(repo_root=repo_root, intent_path=path, template=template)) + intent_id = str(template.get("intent_id") or path.stem) + rows.append( + classify_template( + repo_root=repo_root, + intent_path=path, + template=template, + profile_evidence=profile_evidence.get(intent_id), + ) + ) by_status = Counter(row.production_status for row in rows) + by_status_v2 = Counter(row.semantic_status_v2() for row in rows) by_domain = Counter(row.domain for row in rows) payload = { "schema_version": "ovk.template_conformance.v1", @@ -293,8 +382,11 @@ def build_conformance_matrix(repo_root: Path, templates_dir: Path | None = None) "required_row_fields": list(REQUIRED_ROW_FIELDS), "required_executable_links": list(REQUIRED_EXECUTABLE_LINKS), "production_statuses": list(STATUS_RANK.keys()), + "conformance_statuses_v2": list(STATUS_RANK_V2.keys()), "counts_by_status": dict(sorted(by_status.items())), + "counts_by_status_v2": dict(sorted(by_status_v2.items())), "counts_by_domain": dict(sorted(by_domain.items())), + "source_profile_evidence": {intent: item.as_dict() for intent, item in sorted(profile_evidence.items())}, "templates": [row.to_dict() for row in rows], } return payload @@ -309,6 +401,7 @@ def validate_matrix(matrix: dict[str, Any]) -> list[str]: if not isinstance(templates, list) or not templates: failures.append("templates must be a non-empty list") return failures + allowed_v2 = set(STATUS_RANK_V2) for index, row in enumerate(templates): if not isinstance(row, dict): failures.append(f"templates[{index}] must be an object") @@ -319,11 +412,24 @@ def validate_matrix(matrix: dict[str, Any]) -> list[str]: status = row.get("production_status") if status not in STATUS_RANK: failures.append(f"templates[{index}] invalid production_status {status!r}") + status_v2 = row.get("conformance_status_v2") + if status_v2 is not None and status_v2 not in allowed_v2: + failures.append(f"templates[{index}] invalid conformance_status_v2 {status_v2!r}") + if status_v2 == "externally_calibrated_strict": + evidence = row.get("source_profile_evidence") or {} + if not evidence.get("strict_eligible"): + failures.append( + f"{row.get('intent_id')}: externally_calibrated_strict requires strict profile evidence" + ) + if status_v2 == "source_profile_strict_eligible": + evidence = row.get("source_profile_evidence") or {} + if not evidence.get("strict_eligible"): + failures.append( + f"{row.get('intent_id')}: source_profile_strict_eligible requires executed profile evidence" + ) missing = row.get("missing_executable_links") or [] if status in {"strict_eligible", "advisory"} and missing and status == "strict_eligible": - failures.append( - f"{row.get('intent_id')}: strict_eligible requires empty missing_executable_links" - ) + failures.append(f"{row.get('intent_id')}: strict_eligible requires empty missing_executable_links") # catalog_only is mandatory when required links are incomplete and no # experimental/advisory catalog path was registered. if ( diff --git a/ovk/core/verified_source.py b/ovk/core/verified_source.py index 3a3d0db..951f662 100644 --- a/ovk/core/verified_source.py +++ b/ovk/core/verified_source.py @@ -1,8 +1,11 @@ -"""Resolve the immutable commit that produced verification metrics. +"""Resolve benchmark vs verified source SHAs for metric provenance. -Badge and summary commits may use ``[skip ci]`` and must not be confused with -the commit that actually ran FormalPR-Bench / pilot metrics. Prefer an explicit -override, then GitHub Actions SHA, then local ``git rev-parse HEAD``. +``benchmark_source_sha`` is the commit whose FormalPR-Bench (or badge) artifacts +were measured. ``verified_source_sha`` is only set when a complete observed +required-workflow set is attested (explicit override or ``OVK_VERIFIED_SOURCE_SHA``). + +Badge-only / ``[skip ci]`` commits must never be labeled verified merely because +``GITHUB_SHA`` or ``git HEAD`` is available. """ from __future__ import annotations @@ -12,19 +15,7 @@ from pathlib import Path -def resolve_verified_source_sha( - *, - explicit: str | None = None, - repo_root: Path | None = None, -) -> str | None: - """Return a full git commit SHA for metric provenance, or None if unknown.""" - if explicit: - value = explicit.strip() - return value or None - for key in ("OVK_VERIFIED_SOURCE_SHA", "GITHUB_SHA"): - value = (os.environ.get(key) or "").strip() - if value: - return value +def _git_head(repo_root: Path | None) -> str | None: root = repo_root or Path(__file__).resolve().parents[1] try: completed = subprocess.run( @@ -41,3 +32,39 @@ def resolve_verified_source_sha( return None value = completed.stdout.strip() return value or None + + +def resolve_benchmark_source_sha( + *, + explicit: str | None = None, + repo_root: Path | None = None, +) -> str | None: + """Return the commit SHA that produced benchmark/badge artifacts, if known.""" + if explicit: + value = explicit.strip() + return value or None + for key in ("OVK_BENCHMARK_SOURCE_SHA", "GITHUB_SHA"): + value = (os.environ.get(key) or "").strip() + if value: + return value + return _git_head(repo_root) + + +def resolve_verified_source_sha( + *, + explicit: str | None = None, + repo_root: Path | None = None, +) -> str | None: + """Return a verified-source SHA only when explicitly attested. + + Does not fall back to ``GITHUB_SHA`` or ``git HEAD``. Those identify the + current checkout / workflow run and belong on ``benchmark_source_sha`` unless + maintainers also set ``OVK_VERIFIED_SOURCE_SHA`` after observing the full + required-workflow set. + """ + del repo_root # retained for API compatibility; verified SHA is never inferred from HEAD + if explicit: + value = explicit.strip() + return value or None + value = (os.environ.get("OVK_VERIFIED_SOURCE_SHA") or "").strip() + return value or None diff --git a/ovk/core/worker_runner.py b/ovk/core/worker_runner.py new file mode 100644 index 0000000..84360c1 --- /dev/null +++ b/ovk/core/worker_runner.py @@ -0,0 +1,258 @@ +"""Run backend evaluators behind an externally enforced worker boundary.""" + +from __future__ import annotations + +import json +import sys +import tempfile +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ovk.core.execution_budget import BackendWorker, WorkerResult +from ovk.core.execution_models import RawBackendExecution, compute_raw_execution_digests + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +@dataclass(frozen=True) +class EvaluatorWorkerOutcome: + """Parsed evaluator output plus worker enforcement metadata.""" + + termination: str + exit_code: int + raw_result: dict[str, Any] + native_execution: bool + timed_out: bool + worker_rejected: bool + stderr: str | None + + +def run_evaluator_in_worker( + worker: BackendWorker, + *, + evaluator_id: str, + payload: dict[str, Any], + timeout_seconds: float, + cwd: Path | None = None, +) -> EvaluatorWorkerOutcome: + """Serialize payload, execute evaluator in a subprocess, and parse JSON output.""" + if timeout_seconds <= 0: + return EvaluatorWorkerOutcome( + termination="timeout", + exit_code=1, + raw_result={"status": "unknown", "reason": "budget timeout"}, + native_execution=False, + timed_out=False, + worker_rejected=True, + stderr=f"non-positive wall-time budget rejected: {timeout_seconds}", + ) + + work_cwd = (cwd or Path.cwd()).resolve() + with tempfile.TemporaryDirectory(prefix="ovk-worker-", dir=str(work_cwd)) as tmp: + payload_path = Path(tmp) / "payload.json" + payload_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + command = ( + sys.executable, + "-m", + "ovk.workers.deterministic_entry", + "--evaluator-id", + evaluator_id, + "--payload-file", + str(payload_path), + ) + result = worker.run( + command, + cwd=work_cwd, + timeout_seconds=float(timeout_seconds), + max_stdout_bytes=2_000_000, + max_stderr_bytes=500_000, + ) + return _parse_worker_result(result, evaluator_id=evaluator_id) + + +def _parse_worker_result(result: WorkerResult, *, evaluator_id: str) -> EvaluatorWorkerOutcome: + if result.timed_out: + return EvaluatorWorkerOutcome( + termination="timeout", + exit_code=1, + raw_result={"status": "unknown", "reason": "worker execution timed out"}, + native_execution=False, + timed_out=True, + worker_rejected=False, + stderr=result.stderr or None, + ) + if result.exit_code is None: + return EvaluatorWorkerOutcome( + termination="tool_error", + exit_code=1, + raw_result={ + "status": "error", + "reason": result.stderr.strip() or "worker rejected execution", + }, + native_execution=False, + timed_out=False, + worker_rejected=True, + stderr=result.stderr or None, + ) + if not result.stdout.strip(): + return EvaluatorWorkerOutcome( + termination="invalid_output", + exit_code=1, + raw_result={ + "status": "error", + "reason": result.stderr.strip() or f"{evaluator_id} returned empty output", + }, + native_execution=False, + timed_out=False, + worker_rejected=False, + stderr=result.stderr or None, + ) + try: + envelope = json.loads(result.stdout) + except json.JSONDecodeError: + return EvaluatorWorkerOutcome( + termination="invalid_output", + exit_code=1, + raw_result={ + "status": "error", + "reason": "worker returned invalid JSON", + }, + native_execution=False, + timed_out=False, + worker_rejected=False, + stderr=result.stderr or None, + ) + if not isinstance(envelope, dict): + return EvaluatorWorkerOutcome( + termination="invalid_output", + exit_code=1, + raw_result={"status": "error", "reason": "worker returned non-object JSON"}, + native_execution=False, + timed_out=False, + worker_rejected=False, + stderr=result.stderr or None, + ) + return EvaluatorWorkerOutcome( + termination=str(envelope.get("termination", "completed")), + exit_code=int(envelope.get("exit_code", result.exit_code or 1)), + raw_result=dict(envelope.get("raw_result") or {}), + native_execution=bool(envelope.get("native_execution", False)), + timed_out=False, + worker_rejected=False, + stderr=result.stderr or None, + ) + + +def raw_execution_from_worker_outcome( + *, + backend: str, + backend_obligation_id: str, + adapter_version: str, + outcome: EvaluatorWorkerOutcome, + started_at: str, + duration_ms: float, +) -> RawBackendExecution: + """Project a worker outcome into ``RawBackendExecution``.""" + raw = RawBackendExecution( + backend=backend, + backend_obligation_id=backend_obligation_id, + termination=outcome.termination, # type: ignore[arg-type] + native_execution=outcome.native_execution, + exit_code=outcome.exit_code, + stderr=outcome.stderr, + raw_result=outcome.raw_result, + started_at=started_at, + finished_at=_utc_now_iso(), + duration_ms=duration_ms, + tool_version=adapter_version, + ) + return raw.model_copy(update=compute_raw_execution_digests(raw)) + + +def require_worker( + worker: BackendWorker | None, + *, + backend: str, + backend_obligation_id: str, + adapter_version: str, +) -> BackendWorker: + """Fail closed when an authoritative adapter is invoked without a worker.""" + if worker is not None: + return worker + started_at = _utc_now_iso() + raise WorkerRequiredError( + raw_execution_from_worker_outcome( + backend=backend, + backend_obligation_id=backend_obligation_id, + adapter_version=adapter_version, + outcome=EvaluatorWorkerOutcome( + termination="tool_error", + exit_code=1, + raw_result={ + "status": "error", + "reason": "authoritative adapter requires BackendWorker; in-process execution forbidden", + }, + native_execution=False, + timed_out=False, + worker_rejected=True, + stderr="missing worker", + ), + started_at=started_at, + duration_ms=0.0, + ) + ) + + +class WorkerRequiredError(Exception): + """Raised when an adapter is invoked without a required worker.""" + + def __init__(self, raw: RawBackendExecution) -> None: + super().__init__("authoritative adapter requires BackendWorker") + self.raw = raw + + +def run_with_required_worker( + worker: BackendWorker | None, + *, + backend: str, + backend_obligation_id: str, + adapter_version: str, + evaluator_id: str, + payload: dict[str, Any], + timeout_seconds: float, + cwd: Path | None = None, +) -> RawBackendExecution: + """Execute an evaluator in a worker or return a fail-closed raw execution.""" + started = time.perf_counter() + started_at = _utc_now_iso() + if worker is None: + try: + require_worker( + None, + backend=backend, + backend_obligation_id=backend_obligation_id, + adapter_version=adapter_version, + ) + except WorkerRequiredError as exc: + return exc.raw + assert worker is not None + outcome = run_evaluator_in_worker( + worker, + evaluator_id=evaluator_id, + payload=payload, + timeout_seconds=timeout_seconds, + cwd=cwd, + ) + return raw_execution_from_worker_outcome( + backend=backend, + backend_obligation_id=backend_obligation_id, + adapter_version=adapter_version, + outcome=outcome, + started_at=started_at, + duration_ms=(time.perf_counter() - started) * 1000.0, + ) diff --git a/ovk/mcp_server.py b/ovk/mcp_server.py index 43e454a..e1ed7a7 100644 --- a/ovk/mcp_server.py +++ b/ovk/mcp_server.py @@ -126,11 +126,7 @@ def explain_result(evidence_bundle: dict[str, Any]) -> dict[str, Any]: from ovk.core.counterexample_translator import repair_hint_for_counterexample bundle = EvidenceBundle.model_validate(evidence_bundle) - counterexamples = [ - counterexample - for evidence in bundle.evidence - for counterexample in evidence.counterexamples - ] + counterexamples = [counterexample for evidence in bundle.evidence for counterexample in evidence.counterexamples] repair_hints = [repair_hint_for_counterexample(item) for item in counterexamples] return { "bundle_id": bundle.bundle_id, @@ -190,12 +186,34 @@ def select_backends( intent_id: str, *, changed_files: list[str] | None = None, + lane: str | None = None, + lane_input: dict[str, Any] | None = None, + repo: str = "unknown/repo", + head_sha: str = "unknown", + base_sha: str | None = None, + policy: dict[str, Any] | None = None, ) -> dict[str, Any]: - """Select backends for one intent using capability manifests and repo memory.""" + """Select backends for one intent using the authoritative routing pipeline when possible.""" + from ovk.core.lane_compiler import INTENT_TO_LANE + from ovk.core.router import route_intent, routing_decision_to_legacy_dict + from ovk.core.routing_pipeline import compile_typed_obligation, route_compiled_obligation + + resolved_lane = lane or INTENT_TO_LANE.get(intent_id) + if resolved_lane and lane_input is not None: + obligation = compile_typed_obligation( + lane=resolved_lane, + data=lane_input, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + decision = route_compiled_obligation(obligation, lane=resolved_lane, policy=policy) + return routing_decision_to_legacy_dict(decision, intent_id=intent_id) + from ovk.core.capabilities import CapabilityRegistry from ovk.core.intent_registry import IntentRegistry from ovk.core.repo_memory import router_historical_priors - from ovk.core.router import route_intent from ovk.core.surface_routing import surface_backend_bonuses intents = IntentRegistry.from_directory(resource_path("templates")) @@ -209,6 +227,7 @@ def select_backends( capabilities.all(), historical_priors=router_historical_priors(), surface_bonuses=bonuses, + policy=policy, ) diff --git a/ovk/workers/__init__.py b/ovk/workers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ovk/workers/deterministic_entry.py b/ovk/workers/deterministic_entry.py new file mode 100644 index 0000000..7ed8b53 --- /dev/null +++ b/ovk/workers/deterministic_entry.py @@ -0,0 +1,25 @@ +"""Subprocess worker entry points for isolated backend evaluation.""" + +from __future__ import annotations + +import argparse +import json +import sys + +from ovk.core.deterministic_evaluators import evaluate_deterministic + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Run an OVK evaluator in an isolated worker process.") + parser.add_argument("--evaluator-id", required=True) + parser.add_argument("--payload-file", required=True) + args = parser.parse_args(argv) + + payload = json.loads(open(args.payload_file, encoding="utf-8").read()) + result = evaluate_deterministic(args.evaluator_id, payload) + sys.stdout.write(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/schemas/adoption.summary.schema.json b/schemas/adoption.summary.schema.json index 253d0e9..f74b72c 100644 --- a/schemas/adoption.summary.schema.json +++ b/schemas/adoption.summary.schema.json @@ -18,7 +18,12 @@ "verified_source_sha": { "type": "string", "minLength": 7, - "description": "Git commit that produced the metrics; not a later [skip ci] badge commit." + "description": "Git commit with a complete observed required-workflow set; never a [skip ci] badge-only commit." + }, + "benchmark_source_sha": { + "type": "string", + "minLength": 7, + "description": "Git commit measured by FormalPR-Bench / badge artifacts." }, "formal_pr_bench": { "type": "object", diff --git a/schemas/verification.evidence.v3.schema.json b/schemas/verification.evidence.v3.schema.json new file mode 100644 index 0000000..b17a6fb --- /dev/null +++ b/schemas/verification.evidence.v3.schema.json @@ -0,0 +1,124 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openverification.dev/schemas/verification.evidence.v3.schema.json", + "title": "Verification Evidence v3", + "type": "object", + "required": [ + "evidence_id", + "schema_version", + "subject", + "intent", + "backend_claims", + "decision", + "obligation_id", + "routing_id", + "material_set_digest", + "compiler", + "materials", + "coverage", + "requested_backends", + "eligible_backends", + "selected_backends", + "attempted_backends", + "executed_backends", + "execution_attempts", + "aggregation_policy", + "routing_enforced" + ], + "properties": { + "evidence_id": { "type": "string", "minLength": 1 }, + "schema_version": { "type": "string", "const": "ovk.evidence.v3" }, + "subject": { + "type": "object", + "required": ["repo", "head_sha"], + "properties": { + "repo": { "type": "string" }, + "pull_request": { "type": ["integer", "string", "null"] }, + "head_sha": { "type": "string" }, + "base_sha": { "type": ["string", "null"] } + }, + "additionalProperties": true + }, + "change_origin": { "type": "object", "additionalProperties": true }, + "intent": { + "type": "object", + "required": ["intent_id", "title"], + "properties": { + "intent_id": { "type": "string" }, + "title": { "type": "string" } + }, + "additionalProperties": true + }, + "backend_claims": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["backend", "guarantee_type", "status"], + "properties": { + "backend": { "type": "string" }, + "guarantee_type": { "type": "string" }, + "status": { "type": "string", "enum": ["pass", "fail", "unknown", "error", "skipped"] }, + "assumptions": { "type": "array", "items": { "type": "string" } }, + "limits": { "type": "array", "items": { "type": "string" } }, + "tool_version": { "type": ["string", "null"] }, + "adapter_version": { "type": ["string", "null"] } + }, + "additionalProperties": true + } + }, + "counterexamples": { "type": "array", "items": { "type": "object", "additionalProperties": true } }, + "generated_artifacts": { "type": "array", "items": { "type": "object", "additionalProperties": true } }, + "decision": { + "type": "object", + "required": ["merge_recommendation", "routing_enforced", "aggregation_reason"], + "properties": { + "merge_recommendation": { + "type": "string", + "enum": ["allow", "block", "require_human_review", "allow_with_warning", "require_stronger_check"] + }, + "human_review_required": { "type": "boolean" }, + "routing_enforced": { "type": "boolean" }, + "aggregation_reason": { "type": "string" }, + "fallback_used": { "type": "boolean" }, + "fallback_accepted": { "type": "boolean" }, + "fallback_cause": { "type": ["string", "null"] } + }, + "additionalProperties": true + }, + "obligation_id": { "type": "string", "minLength": 1 }, + "routing_id": { "type": "string", "minLength": 1 }, + "material_set_digest": { "type": "string", "minLength": 1 }, + "compiler": { + "type": "object", + "required": ["compiler_id", "compiler_version"], + "properties": { + "compiler_id": { "type": "string" }, + "compiler_version": { "type": "string" } + }, + "additionalProperties": true + }, + "materials": { + "type": "array", + "items": { + "type": "object", + "required": ["material_id", "sha256"], + "properties": { + "material_id": { "type": "string" }, + "sha256": { "type": "string" } + }, + "additionalProperties": true + } + }, + "coverage": { "type": "object", "additionalProperties": true }, + "requested_backends": { "type": "array", "items": { "type": "string" } }, + "eligible_backends": { "type": "array", "items": { "type": "string" } }, + "selected_backends": { "type": "array", "items": { "type": "string" } }, + "attempted_backends": { "type": "array", "items": { "type": "string" } }, + "executed_backends": { "type": "array", "items": { "type": "string" } }, + "execution_attempts": { "type": "array", "items": { "type": "object", "additionalProperties": true } }, + "aggregation_policy": { "type": "string", "minLength": 1 }, + "routing_enforced": { "type": "boolean" } + }, + "additionalProperties": true +} diff --git a/scripts/build_template_conformance.py b/scripts/build_template_conformance.py index 5a172e0..b4ba85b 100644 --- a/scripts/build_template_conformance.py +++ b/scripts/build_template_conformance.py @@ -49,7 +49,8 @@ def main() -> int: print( f"template conformance: {matrix['template_count']} templates -> {output}" f" (strict_eligible={matrix['counts_by_status'].get('strict_eligible', 0)}," - f" catalog_only={matrix['counts_by_status'].get('catalog_only', 0)})" + f" catalog_only={matrix['counts_by_status'].get('catalog_only', 0)}," + f" source_profile_strict_eligible={matrix.get('counts_by_status_v2', {}).get('source_profile_strict_eligible', 0)})" ) if failures: for failure in failures: diff --git a/scripts/collect_workflow_evidence.py b/scripts/collect_workflow_evidence.py new file mode 100644 index 0000000..7b00d0d --- /dev/null +++ b/scripts/collect_workflow_evidence.py @@ -0,0 +1,109 @@ +"""Collect GitHub Actions workflow run IDs and digests for attributable publication. + +Requires network + ``gh`` auth. Does not push or mutate remotes. +When Actions are unavailable, exits non-zero with an explicit blocker message. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +REQUIRED_WORKFLOW_NAMES = ( + "CI", + "Native Tier 1", + "Release", + "Bench", +) + + +def _run_gh(args: list[str]) -> tuple[int, str, str]: + try: + completed = subprocess.run( + ["gh", *args], + capture_output=True, + text=True, + check=False, + timeout=60, + ) + except (OSError, subprocess.SubprocessError) as exc: + return 1, "", str(exc) + return completed.returncode, completed.stdout, completed.stderr + + +def collect_for_sha(*, repo: str, sha: str, limit: int = 30) -> dict[str, Any]: + code, stdout, stderr = _run_gh( + [ + "run", + "list", + "--repo", + repo, + "--commit", + sha, + "--limit", + str(limit), + "--json", + "databaseId,displayTitle,workflowName,status,conclusion,url,headSha,createdAt", + ] + ) + if code != 0: + return { + "ok": False, + "blocker": "gh_run_list_failed", + "detail": stderr.strip() or stdout.strip() or "gh run list failed", + "benchmark_source_sha": sha, + "verified_source_sha": None, + "collected_at": datetime.now(timezone.utc).isoformat(), + "runs": [], + } + runs = json.loads(stdout or "[]") + by_workflow: dict[str, list[dict[str, Any]]] = {} + for run in runs: + name = str(run.get("workflowName") or "unknown") + by_workflow.setdefault(name, []).append(run) + observed = sorted(by_workflow) + # verified_source_sha is only set when callers confirm the full required set. + return { + "ok": True, + "blocker": None, + "benchmark_source_sha": sha, + "verified_source_sha": None, + "required_workflow_names": list(REQUIRED_WORKFLOW_NAMES), + "observed_workflow_names": observed, + "collected_at": datetime.now(timezone.utc).isoformat(), + "runs": runs, + "note": ( + "Cite benchmark_source_sha for measurement identity. " + "Set verified_source_sha only after maintainers confirm the complete " + "required-workflow set on this commit." + ), + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Collect workflow evidence for a source SHA") + parser.add_argument("--repo", default="fraware/open-verification-kernel") + parser.add_argument("--sha", required=True, help="Source commit SHA") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + payload = collect_for_sha(repo=args.repo, sha=args.sha) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + if not payload.get("ok"): + print(f"blocked: {payload.get('blocker')}: {payload.get('detail')}", file=sys.stderr) + return 2 + print( + f"collected {len(payload.get('runs') or [])} runs for " + f"benchmark_source_sha={args.sha}; verified_source_sha left unset" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/digest_holdout_predictions.py b/scripts/digest_holdout_predictions.py new file mode 100644 index 0000000..a8b0467 --- /dev/null +++ b/scripts/digest_holdout_predictions.py @@ -0,0 +1,105 @@ +"""Digest and validate label-free holdout predictions (Sprint 8). + +Predictions are produced from the RC artifact without protected labels, then +digested (and optionally signed) before a separate evaluator consumes them. + +Case ids may appear in predictions (needed for scoring). Protected *labels* / +ground-truth fields must not. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any + +_LABEL_FORBIDDEN_SUBSTRINGS = ( + "expected_status", + "expected_merge_recommendation", + "ground_truth_class", + "true_positive_unsafe", + "true_negative_safe", + "corpus/labels", + "labels_dir", + "ground_truth", + "expected_outcome", +) + +_TOP_LEVEL_FORBIDDEN = frozenset( + { + "labels", + "expected_status", + "ground_truth_class", + "corpus_labels", + "expected_merge_recommendation", + } +) + +_CASE_FORBIDDEN = frozenset( + { + "expected_status", + "ground_truth_class", + "label", + "labels", + "expected_merge_recommendation", + } +) + + +def _fail(msg: str) -> None: + raise SystemExit(f"fail-closed: {msg}") + + +def assert_predictions_label_free(payload: Any) -> None: + """Refuse predictions that embed protected labels or case-ground-truth fields.""" + text = json.dumps(payload, sort_keys=True) + for token in _LABEL_FORBIDDEN_SUBSTRINGS: + if token in text: + _fail(f"predictions contain protected token {token!r}") + if isinstance(payload, dict): + for key in _TOP_LEVEL_FORBIDDEN: + if key in payload: + _fail(f"predictions must not include top-level key {key!r}") + cases = payload.get("cases") or payload.get("predictions") + if isinstance(cases, list): + for index, item in enumerate(cases): + if not isinstance(item, dict): + continue + for key in _CASE_FORBIDDEN: + if key in item: + _fail(f"predictions[{index}] must not include {key!r}") + + +def digest_predictions_file(path: Path) -> dict[str, Any]: + raw = path.read_bytes() + payload = json.loads(raw.decode("utf-8")) + assert_predictions_label_free(payload) + digest = hashlib.sha256(raw).hexdigest() + return { + "schema_version": "ovk.holdout.predictions_digest.v1", + "predictions_path": path.as_posix(), + "sha256": digest, + "byte_length": len(raw), + "label_free": True, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Digest label-free holdout predictions") + parser.add_argument("--predictions", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + if not args.predictions.is_file(): + _fail(f"predictions file not found: {args.predictions}") + record = digest_predictions_file(args.predictions) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + print(f"predictions digest ok: sha256={record['sha256']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/expand_ovk_library.py b/scripts/expand_ovk_library.py index 1510643..1590c9c 100644 --- a/scripts/expand_ovk_library.py +++ b/scripts/expand_ovk_library.py @@ -17,7 +17,12 @@ DOMAIN_CONFIG: dict[str, dict[str, Any]] = { "authorization": { "kinds": ["access_control", "safety", "invariant"], - "failure_modes": ["middleware_skipped", "route_group_unprotected", "policy_default_allow", "role_check_removed"], + "failure_modes": [ + "middleware_skipped", + "route_group_unprotected", + "policy_default_allow", + "role_check_removed", + ], "evidence": [ {"kind": "policy_check", "minimum_confidence": "medium"}, {"kind": "smt_counterexample", "minimum_confidence": "medium"}, @@ -93,7 +98,11 @@ {"kind": "model_check", "minimum_confidence": "high"}, {"kind": "smt_counterexample", "minimum_confidence": "medium"}, ], - "risk": {"severity": "critical", "likelihood": "medium", "rationale": "Memory safety regressions in auth paths are exploitable."}, + "risk": { + "severity": "critical", + "likelihood": "medium", + "rationale": "Memory safety regressions in auth paths are exploitable.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -113,7 +122,11 @@ }, "failure_modes": ["policy_default_allow", "wildcard_principal", "privilege_escalation"], "acceptable_evidence": [{"kind": "policy_check", "minimum_confidence": "high"}], - "risk": {"severity": "critical", "likelihood": "medium", "rationale": "IAM regressions expose cloud control planes."}, + "risk": { + "severity": "critical", + "likelihood": "medium", + "rationale": "IAM regressions expose cloud control planes.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -133,7 +146,11 @@ }, "failure_modes": ["public_exposure", "lateral_movement_path"], "acceptable_evidence": [{"kind": "topology_model", "minimum_confidence": "medium"}], - "risk": {"severity": "high", "likelihood": "medium", "rationale": "Topology regressions enable data exfiltration."}, + "risk": { + "severity": "high", + "likelihood": "medium", + "rationale": "Topology regressions enable data exfiltration.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -153,7 +170,11 @@ }, "failure_modes": ["skipped_approval", "invalid_state_transition"], "acceptable_evidence": [{"kind": "model_check", "minimum_confidence": "high"}], - "risk": {"severity": "high", "likelihood": "medium", "rationale": "Skipped approvals enable unreviewed production changes."}, + "risk": { + "severity": "high", + "likelihood": "medium", + "rationale": "Skipped approvals enable unreviewed production changes.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -173,7 +194,11 @@ }, "failure_modes": ["buffer_overflow", "use_after_free"], "acceptable_evidence": [{"kind": "memory_model", "minimum_confidence": "high"}], - "risk": {"severity": "critical", "likelihood": "low", "rationale": "Memory corruption can leak cross-tenant data."}, + "risk": { + "severity": "critical", + "likelihood": "low", + "rationale": "Memory corruption can leak cross-tenant data.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -193,7 +218,11 @@ }, "failure_modes": ["self_approval", "gate_removal", "bot_bypass"], "acceptable_evidence": [{"kind": "policy_check", "minimum_confidence": "high"}], - "risk": {"severity": "critical", "likelihood": "high", "rationale": "Agents must not weaken their own verification gates."}, + "risk": { + "severity": "critical", + "likelihood": "high", + "rationale": "Agents must not weaken their own verification gates.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -213,7 +242,11 @@ }, "failure_modes": ["secret_in_untrusted_context", "workflow_permission_escalation"], "acceptable_evidence": [{"kind": "smt_counterexample", "minimum_confidence": "medium"}], - "risk": {"severity": "critical", "likelihood": "medium", "rationale": "Secret exposure in CI is a common agent regression."}, + "risk": { + "severity": "critical", + "likelihood": "medium", + "rationale": "Secret exposure in CI is a common agent regression.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -232,7 +265,11 @@ }, "failure_modes": ["cross_account_trust", "resource_policy_wildcard"], "acceptable_evidence": [{"kind": "policy_check", "minimum_confidence": "high"}], - "risk": {"severity": "high", "likelihood": "medium", "rationale": "Cross-account leaks expand blast radius."}, + "risk": { + "severity": "high", + "likelihood": "medium", + "rationale": "Cross-account leaks expand blast radius.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -270,7 +307,11 @@ }, "failure_modes": ["rollback_bypass", "invalid_state_transition"], "acceptable_evidence": [{"kind": "model_check", "minimum_confidence": "medium"}], - "risk": {"severity": "high", "likelihood": "low", "rationale": "Unsafe rollbacks can reintroduce known failures."}, + "risk": { + "severity": "high", + "likelihood": "low", + "rationale": "Unsafe rollbacks can reintroduce known failures.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -308,7 +349,11 @@ }, "failure_modes": ["privilege_escalation", "self_approval"], "acceptable_evidence": [{"kind": "proof", "minimum_confidence": "high"}], - "risk": {"severity": "critical", "likelihood": "medium", "rationale": "Authority escalation enables autonomous harm."}, + "risk": { + "severity": "critical", + "likelihood": "medium", + "rationale": "Authority escalation enables autonomous harm.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -327,7 +372,11 @@ }, "failure_modes": ["gate_removal", "unsigned_artifact_publish"], "acceptable_evidence": [{"kind": "proof", "minimum_confidence": "medium"}], - "risk": {"severity": "high", "likelihood": "medium", "rationale": "Weakened build gates enable supply-chain regressions."}, + "risk": { + "severity": "high", + "likelihood": "medium", + "rationale": "Weakened build gates enable supply-chain regressions.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -346,7 +395,11 @@ }, "failure_modes": ["privilege_escalation", "policy_default_allow"], "acceptable_evidence": [{"kind": "policy_check", "minimum_confidence": "high"}], - "risk": {"severity": "critical", "likelihood": "medium", "rationale": "Privilege escalation is a primary auth regression class."}, + "risk": { + "severity": "critical", + "likelihood": "medium", + "rationale": "Privilege escalation is a primary auth regression class.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -365,7 +418,11 @@ }, "failure_modes": ["public_exposure", "lateral_movement_path"], "acceptable_evidence": [{"kind": "topology_model", "minimum_confidence": "medium"}], - "risk": {"severity": "high", "likelihood": "medium", "rationale": "Public egress from sensitive tiers enables exfiltration."}, + "risk": { + "severity": "high", + "likelihood": "medium", + "rationale": "Public egress from sensitive tiers enables exfiltration.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, diff --git a/scripts/render_bench_badge.py b/scripts/render_bench_badge.py index 491612e..5c7e23e 100644 --- a/scripts/render_bench_badge.py +++ b/scripts/render_bench_badge.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any -from ovk.core.verified_source import resolve_verified_source_sha +from ovk.core.verified_source import resolve_benchmark_source_sha, resolve_verified_source_sha ROOT = Path(__file__).resolve().parents[1] DEFAULT_LEADERBOARD = ROOT / ".verification" / "formal-pr-bench-leaderboard.json" @@ -30,47 +30,52 @@ def badge_color(cases_passed: int, cases_total: int) -> str: def render_badge( leaderboard: dict[str, Any], *, + benchmark_source_sha: str | None = None, verified_source_sha: str | None = None, ) -> dict[str, Any]: """Build shields.io endpoint badge payload. - ``verified_source_sha`` records the commit that produced the leaderboard, - not a later ``[skip ci]`` badge-only commit. + ``benchmark_source_sha`` is the commit measured by FormalPR-Bench. + ``verified_source_sha`` is set only when a complete required-workflow set was + observed for that source; badge-only commits must not be labeled verified. """ summary = leaderboard.get("summary", {}) cases_total = int(summary.get("cases_total", 0)) cases_passed = int(summary.get("cases_passed", 0)) rate = (cases_passed / cases_total * 100.0) if cases_total else 0.0 - sha = verified_source_sha or resolve_verified_source_sha() + bench_sha = benchmark_source_sha or resolve_benchmark_source_sha() + verified_sha = verified_source_sha or resolve_verified_source_sha() payload: dict[str, Any] = { "schemaVersion": 1, "label": "FormalPR-Bench", "message": f"{cases_passed}/{cases_total} ({rate:.0f}%)", "color": badge_color(cases_passed, cases_total), } - if sha: - payload["verified_source_sha"] = sha + if bench_sha: + payload["benchmark_source_sha"] = bench_sha + if verified_sha: + payload["verified_source_sha"] = verified_sha return payload def render_summary( leaderboard: dict[str, Any], *, + benchmark_source_sha: str | None = None, verified_source_sha: str | None = None, ) -> dict[str, Any]: """Build trimmed public summary for docs and README links.""" summary = leaderboard.get("summary", {}) timing = leaderboard.get("timing_ms", {}) - sha = verified_source_sha or resolve_verified_source_sha() + bench_sha = benchmark_source_sha or resolve_benchmark_source_sha() + verified_sha = verified_source_sha or resolve_verified_source_sha() payload: dict[str, Any] = { "schema_version": "formal_pr_bench.summary.v1", "generated_from": leaderboard.get("schema_version", "formal_pr_bench.leaderboard.v1"), "cases_total": summary.get("cases_total", 0), "cases_passed": summary.get("cases_passed", 0), "pass_rate": ( - summary.get("cases_passed", 0) / summary.get("cases_total", 1) - if summary.get("cases_total") - else 0.0 + summary.get("cases_passed", 0) / summary.get("cases_total", 1) if summary.get("cases_total") else 0.0 ), "merge_decision_accuracy": summary.get("merge_decision_accuracy"), "status_accuracy": summary.get("status_accuracy"), @@ -86,12 +91,15 @@ def render_summary( "max": timing.get("max"), }, } - if sha: - payload["verified_source_sha"] = sha - payload["provenance_note"] = ( - "Cite verified_source_sha (and its CI run) for health claims; " - "do not treat a later [skip ci] badge commit as the verified source." - ) + if bench_sha: + payload["benchmark_source_sha"] = bench_sha + if verified_sha: + payload["verified_source_sha"] = verified_sha + payload["provenance_note"] = ( + "Cite benchmark_source_sha for FormalPR-Bench measurement identity. " + "Cite verified_source_sha only when a complete required-workflow set was " + "observed for that commit; do not treat a later [skip ci] badge commit as verified." + ) return payload @@ -99,13 +107,23 @@ def write_outputs( leaderboard_path: Path, *, dry_run: bool = False, + benchmark_source_sha: str | None = None, verified_source_sha: str | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: """Read leaderboard and write badge + summary files.""" leaderboard = json.loads(leaderboard_path.read_text(encoding="utf-8")) - sha = verified_source_sha or resolve_verified_source_sha() - badge = render_badge(leaderboard, verified_source_sha=sha) - summary = render_summary(leaderboard, verified_source_sha=sha) + bench_sha = benchmark_source_sha or resolve_benchmark_source_sha() + verified_sha = verified_source_sha or resolve_verified_source_sha() + badge = render_badge( + leaderboard, + benchmark_source_sha=bench_sha, + verified_source_sha=verified_sha, + ) + summary = render_summary( + leaderboard, + benchmark_source_sha=bench_sha, + verified_source_sha=verified_sha, + ) if not dry_run: BADGE_PATH.parent.mkdir(parents=True, exist_ok=True) BADGE_PATH.write_text(json.dumps(badge, indent=2) + "\n", encoding="utf-8") @@ -121,10 +139,15 @@ def main() -> int: default=DEFAULT_LEADERBOARD, help="Path to formal-pr-bench-leaderboard.json", ) + parser.add_argument( + "--benchmark-source-sha", + default=None, + help="Commit measured by FormalPR-Bench (defaults to GITHUB_SHA / git HEAD)", + ) parser.add_argument( "--verified-source-sha", default=None, - help="Commit that produced the leaderboard (defaults to GITHUB_SHA / git HEAD)", + help="Commit with a complete observed required-workflow set (optional)", ) parser.add_argument("--dry-run", action="store_true", help="Compute outputs without writing files") args = parser.parse_args() @@ -134,12 +157,15 @@ def main() -> int: badge, summary = write_outputs( args.leaderboard, dry_run=args.dry_run, + benchmark_source_sha=args.benchmark_source_sha, verified_source_sha=args.verified_source_sha, ) if args.dry_run: print(json.dumps({"badge": badge, "summary": summary}, indent=2)) else: print(f"wrote {BADGE_PATH.relative_to(ROOT)} and {SUMMARY_PATH.relative_to(ROOT)}") + if summary.get("benchmark_source_sha"): + print(f"benchmark_source_sha={summary['benchmark_source_sha']}") if summary.get("verified_source_sha"): print(f"verified_source_sha={summary['verified_source_sha']}") return 0 diff --git a/scripts/render_pilot_metrics.py b/scripts/render_pilot_metrics.py index eabddcb..d99b4f9 100644 --- a/scripts/render_pilot_metrics.py +++ b/scripts/render_pilot_metrics.py @@ -152,7 +152,9 @@ def parse_args() -> argparse.Namespace: default=EXTERNAL_PILOTS_REGISTRY_PATH, help="External pilots registry JSON (default: docs/benchmarks/external-pilots-registry.json)", ) - parser.add_argument("--output", type=Path, default=ADOPTION_SUMMARY_PATH, help="Output path for adoption-summary.json") + parser.add_argument( + "--output", type=Path, default=ADOPTION_SUMMARY_PATH, help="Output path for adoption-summary.json" + ) parser.add_argument( "--verified-source-sha", default=None, diff --git a/scripts/run_formalpr_holdout.py b/scripts/run_formalpr_holdout.py index aafb9ef..19f6ab3 100644 --- a/scripts/run_formalpr_holdout.py +++ b/scripts/run_formalpr_holdout.py @@ -60,13 +60,23 @@ def _fail(msg: str) -> None: raise SystemExit(f"fail-closed: {msg}") +_ALLOWED_META_KEYS = frozenset( + { + "labels_emitted", + "case_ids_emitted", + "fail_closed", + "sanitizer_version", + } +) + + def _walk_keys(value: Any, *, path: str = "$") -> list[tuple[str, str]]: findings: list[tuple[str, str]] = [] if isinstance(value, dict): for key, child in value.items(): key_text = str(key) lowered = key_text.lower() - if any(fragment in lowered for fragment in FORBIDDEN_KEY_FRAGMENTS): + if key_text not in _ALLOWED_META_KEYS and any(fragment in lowered for fragment in FORBIDDEN_KEY_FRAGMENTS): findings.append((f"{path}.{key_text}", key_text)) findings.extend(_walk_keys(child, path=f"{path}.{key_text}")) elif isinstance(value, list): @@ -138,6 +148,95 @@ def verify_asset_digest(path: Path, expected_sha256: str | None) -> str: return actual +_SHA256_RE = __import__("re").compile(r"^[0-9a-fA-F]{64}$") +_TOKEN_ENV_KEYS = frozenset( + { + "HOLDOUT_DOWNLOAD_TOKEN", + "GITHUB_TOKEN", + "GH_TOKEN", + "ACTIONS_RUNTIME_TOKEN", + } +) +_REQUIRED_AGGREGATE_KEYS = ( + "schema_version", + "benchmark", + "holdout_release_tag", + "ovk_commit_sha", + "cases_scored", + "lanes", + "leakage_guard", +) + + +def verify_asset_sha256(path: Path, expected_sha256: str) -> str: + if not _SHA256_RE.match(expected_sha256): + _fail("asset SHA-256 must be a 64-character hex digest") + digest = sha256_file(path) + if digest.lower() != expected_sha256.lower(): + _fail("asset SHA-256 mismatch") + return digest + + +def validate_aggregate_schema(payload: dict[str, Any]) -> None: + """Fail-closed structural validation for holdout aggregate metrics.""" + for key in _REQUIRED_AGGREGATE_KEYS: + if key not in payload: + _fail(f"aggregate missing required key {key!r}") + if payload.get("schema_version") != "formalpr_holdout.aggregate_metrics.v1": + _fail("unexpected aggregate schema_version") + if payload.get("benchmark") != "FormalPR-Holdout": + _fail("unexpected aggregate benchmark") + lanes = payload.get("lanes") + if not isinstance(lanes, dict) or not lanes: + _fail("aggregate lanes must be a non-empty object") + for lane, metrics in lanes.items(): + if not isinstance(metrics, dict): + _fail(f"lane {lane!r} metrics must be an object") + for key in ( + "precision", + "recall", + "false_positive_rate", + "missed_detection_rate", + "unknown_rate", + "coverage_completeness", + "counterexample_correctness", + "selected_backend_execution_correctness", + "runtime_ms", + ): + if key not in metrics: + _fail(f"missing {key} in lane {lane}") + assert_aggregate_safe(payload) + + +def _isolated_eval_env() -> dict[str, str]: + """Build an evaluator environment without download/GitHub tokens.""" + allow = { + "PATH", + "PATHEXT", + "SYSTEMROOT", + "TEMP", + "TMP", + "TMPDIR", + "HOME", + "USERPROFILE", + "LANG", + "LC_ALL", + "PYTHONPATH", + "VIRTUAL_ENV", + "PYTHONNOUSERSITE", + "PYTHONUTF8", + "PYTHONIOENCODING", + } + env = {k: v for k, v in os.environ.items() if k.upper() in allow} + env["PYTHONNOUSERSITE"] = "1" + for key in list(env): + if key.upper() in _TOKEN_ENV_KEYS: + env.pop(key, None) + for denied in _TOKEN_ENV_KEYS: + env.pop(denied, None) + return env + + def download_release_asset( *, repo: str, @@ -180,12 +279,12 @@ def download_release_asset( def _safe_member_target(dest: Path, member_name: str) -> Path: pure = PurePosixPath(member_name) if pure.is_absolute() or any(part in {"", ".", ".."} for part in pure.parts): - _fail(f"unsafe archive member path: {member_name!r}") + _fail(f"unsafe archive member path (unsafe path / path traversal): {member_name!r}") target = (dest / Path(*pure.parts)).resolve() try: target.relative_to(dest.resolve()) except ValueError: - _fail(f"archive member escapes extraction root: {member_name!r}") + _fail(f"archive member escapes extraction root (path traversal forbidden): {member_name!r}") return target @@ -198,7 +297,7 @@ def extract_tarball(tarball: Path, dest: Path) -> Path: for member in members: target = _safe_member_target(dest, member.name) if member.issym() or member.islnk() or member.isdev() or member.isfifo(): - _fail(f"archive contains forbidden special member: {member.name!r}") + _fail(f"forbidden special member: link member forbidden: {member.name!r}") if member.isdir(): target.mkdir(parents=True, exist_ok=True) continue @@ -282,7 +381,7 @@ def run_harness( proc = subprocess.run( cmd, cwd=str(release_root), - env=_harness_environment(output.parent / "harness-home"), + env=_isolated_eval_env(), capture_output=True, text=True, check=False, @@ -308,8 +407,8 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument( "--asset-sha256", - default=None, - help="Expected immutable SHA-256. Required for remote downloads.", + required=True, + help="Immutable SHA-256 hex digest of the release asset (required)", ) parser.add_argument( "--artifact", @@ -331,6 +430,8 @@ def main(argv: list[str] | None = None) -> int: token = os.environ.get("HOLDOUT_DOWNLOAD_TOKEN") or os.environ.get("GITHUB_TOKEN") asset_name = args.asset_name or f"FormalPR-Holdout-{args.tag}.tar.gz" expected_digest = args.asset_sha256 or os.environ.get("HOLDOUT_ASSET_SHA256") + if not expected_digest: + _fail("HOLDOUT_ASSET_SHA256 or --asset-sha256 is required") with tempfile.TemporaryDirectory(prefix="ovk-holdout-") as tmp: tmp_path = Path(tmp) @@ -353,8 +454,13 @@ def main(argv: list[str] | None = None) -> int: dest=tmp_path / asset_name, token=token, ) - verify_asset_digest(tarball, expected_digest) + verify_asset_sha256(tarball, expected_digest) release_root = extract_tarball(tarball, tmp_path / "extract") + # Predictions must be label-free before the evaluator sees them. + pred_payload = json.loads(args.predictions.read_text(encoding="utf-8")) + from scripts.digest_holdout_predictions import assert_predictions_label_free + + assert_predictions_label_free(pred_payload) payload = run_harness( release_root=release_root, predictions=args.predictions, diff --git a/scripts/sync_real_diff_cases.py b/scripts/sync_real_diff_cases.py index 5fb56df..4807d5c 100644 --- a/scripts/sync_real_diff_cases.py +++ b/scripts/sync_real_diff_cases.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Generate benchmarks/formal_pr_bench/real_diff_cases.json from manifest.""" + from __future__ import annotations import json diff --git a/scripts/write_artifact_manifest.py b/scripts/write_artifact_manifest.py index 2143abc..7052fa3 100644 --- a/scripts/write_artifact_manifest.py +++ b/scripts/write_artifact_manifest.py @@ -13,7 +13,9 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Write an OVK artifact manifest") parser.add_argument("--artifact", action="append", default=[], help="Artifact path to include") - parser.add_argument("--kind", action="append", default=[], help="Optional artifact kind, aligned with --artifact order") + parser.add_argument( + "--kind", action="append", default=[], help="Optional artifact kind, aligned with --artifact order" + ) parser.add_argument("--root", type=Path, default=Path("."), help="Root used for relative paths") parser.add_argument("--output", type=Path, default=Path("ovk-artifact-manifest.json")) return parser.parse_args() @@ -25,8 +27,7 @@ def main() -> int: raise SystemExit("--kind must be supplied once per --artifact when used") kinds = args.kind or ["artifact"] * len(args.artifact) entries = [ - artifact_entry(Path(path), kind=kind, root=args.root) - for path, kind in zip(args.artifact, kinds, strict=True) + artifact_entry(Path(path), kind=kind, root=args.root) for path, kind in zip(args.artifact, kinds, strict=True) ] manifest = build_artifact_manifest(entries) args.output.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") diff --git a/tests/test_adapter_isolation_r2_pr8.py b/tests/test_adapter_isolation_r2_pr8.py new file mode 100644 index 0000000..8d5cc5f --- /dev/null +++ b/tests/test_adapter_isolation_r2_pr8.py @@ -0,0 +1,83 @@ +"""PR8 — native adapter worker isolation tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from ovk.adapters.authorization.z3_adapter import Z3NativeAuthorizationAdapter +from ovk.adapters.self_protection.opa_adapter import OpaNativeSelfProtectionAdapter +from ovk.core.execution_budget import WorkerResult +from ovk.core.execution_models import ExecutionBudget + + +def _budget() -> ExecutionBudget: + return ExecutionBudget( + total_wall_time_seconds=30, + per_backend_wall_time_seconds=30, + max_memory_mb=512, + max_parallel_backends=1, + allow_network=False, + allow_repository_write=False, + ) + + +def test_z3_adapter_requires_worker() -> None: + adapter = Z3NativeAuthorizationAdapter() + from ovk.core.authorization_compiler import compile_authorization_obligation + from ovk.core.routing_pipeline import route_compiled_obligation + + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + obligation = compile_authorization_obligation(data, repo="r", head_sha="h") + routing = route_compiled_obligation(obligation, lane="authorization") + compiled = adapter.compile(obligation, routing) + raw = adapter.run(compiled, _budget(), worker=None) + assert raw.termination == "tool_error" + + +def test_opa_adapter_requires_worker() -> None: + adapter = OpaNativeSelfProtectionAdapter() + from ovk.core.self_protection_compiler import compile_self_protection_obligation + from ovk.core.routing_pipeline import route_compiled_obligation + + data = {"before": {"required_check": "ovk"}, "after": {"required_check": None}} + obligation = compile_self_protection_obligation(data, repo="r", head_sha="h", metadata_trusted=True) + routing = route_compiled_obligation(obligation, lane="self_protection") + compiled = adapter.compile(obligation, routing) + raw = adapter.run(compiled, _budget(), worker=None) + assert raw.termination == "tool_error" + + +def test_worker_timeout_enforced_for_z3_not_post_hoc() -> None: + adapter = Z3NativeAuthorizationAdapter() + from ovk.core.authorization_compiler import compile_authorization_obligation + from ovk.core.routing_pipeline import route_compiled_obligation + + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + obligation = compile_authorization_obligation(data, repo="r", head_sha="h") + routing = route_compiled_obligation(obligation, lane="authorization") + compiled = adapter.compile(obligation, routing) + + class SlowWorker: + def run( + self, + command, + *, + cwd, + env=None, + timeout_seconds=30.0, + max_stdout_bytes=1_000_000, + max_stderr_bytes=1_000_000, + ): + return WorkerResult( + exit_code=None, + timed_out=True, + stdout="", + stderr="timed out", + cwd=str(cwd), + command=tuple(command), + ) + + raw = adapter.run(compiled, _budget(), worker=SlowWorker()) + assert raw.termination == "timeout" + assert raw.raw_result.get("reason") == "worker execution timed out" diff --git a/tests/test_adversarial_control_plane.py b/tests/test_adversarial_control_plane.py index 72a1ad9..02bd0cf 100644 --- a/tests/test_adversarial_control_plane.py +++ b/tests/test_adversarial_control_plane.py @@ -197,9 +197,7 @@ def test_malformed_backend_output_maps_to_error_or_invalid() -> None: config=RoutingConfig(prefer_deterministic=True, max_selected_backends=1, accept_partial_primary=True), policy={"budget": {"allowed_backends": ["authorization-deterministic"]}}, ) - record = BackendControlPlane(use_hardened_cache=False).execute( - obligation, routing, registry=registry, cache=None - ) + record = BackendControlPlane(use_hardened_cache=False).execute(obligation, routing, registry=registry, cache=None) assert record.results assert record.results[0].status in {VerificationStatus.UNKNOWN, VerificationStatus.ERROR} assert record.attempts[0].termination in {"invalid_output", "completed", "tool_error"} @@ -229,9 +227,7 @@ def test_incomplete_abstraction_cannot_allow_under_strict() -> None: ), policy={"budget": {"allowed_backends": ["authorization-deterministic"]}}, ) - record = BackendControlPlane(use_hardened_cache=False).execute( - obligation, routing, registry=registry, cache=None - ) + record = BackendControlPlane(use_hardened_cache=False).execute(obligation, routing, registry=registry, cache=None) evidence = execution_record_to_evidence(record, routing_enforced=True) assert evidence.decision["merge_recommendation"] != "allow" @@ -266,9 +262,7 @@ def test_backend_timeout_never_deterministic_pass_fallback() -> None: ] } ) - record = BackendControlPlane(use_hardened_cache=False).execute( - obligation, routing, registry=registry, cache=None - ) + record = BackendControlPlane(use_hardened_cache=False).execute(obligation, routing, registry=registry, cache=None) assert record.aggregate_status != VerificationStatus.PASS assert record.merge_recommendation != MergeRecommendation.ALLOW assert record.attempts[0].termination == "timeout" @@ -311,9 +305,7 @@ def test_render_and_attestation_expose_enforced_fields() -> None: config=RoutingConfig(prefer_deterministic=True, max_selected_backends=1), policy={"budget": {"allowed_backends": ["authorization-deterministic"]}}, ) - record = BackendControlPlane(use_hardened_cache=False).execute( - obligation, routing, registry=registry, cache=None - ) + record = BackendControlPlane(use_hardened_cache=False).execute(obligation, routing, registry=registry, cache=None) evidence = execution_record_to_evidence(record, routing_enforced=True, schema_version="ovk.evidence.v2") markdown = render_evidence_markdown(evidence) assert "Compiler:" in markdown diff --git a/tests/test_auth_obligation_cli.py b/tests/test_auth_obligation_cli.py index a1a3143..7fe9520 100644 --- a/tests/test_auth_obligation_cli.py +++ b/tests/test_auth_obligation_cli.py @@ -99,4 +99,3 @@ def test_auth_obligation_cli_writes_quality_output(tmp_path: Path) -> None: payload = json.loads(quality.read_text(encoding="utf-8")) assert payload["schema_version"] == "ovk.evidence_quality.v1" assert payload["passed"] is True - diff --git a/tests/test_authorization_compilers.py b/tests/test_authorization_compilers.py index 3613a18..44c22eb 100644 --- a/tests/test_authorization_compilers.py +++ b/tests/test_authorization_compilers.py @@ -161,7 +161,10 @@ def admin(): def test_acceptance_corpus_meets_program_targets() -> None: cases = build_corpus(meet_targets=True) - by_framework = {"fastapi": {"pass": 0, "fail": 0, "incomplete": 0}, "express": {"pass": 0, "fail": 0, "incomplete": 0}} + by_framework = { + "fastapi": {"pass": 0, "fail": 0, "incomplete": 0}, + "express": {"pass": 0, "fail": 0, "incomplete": 0}, + } for case in cases: classified = classify_case(case) assert classified == case.category, f"{case.case_id} expected {case.category} got {classified}" diff --git a/tests/test_authorization_enforcement.py b/tests/test_authorization_enforcement.py index 08d87b2..a53164b 100644 --- a/tests/test_authorization_enforcement.py +++ b/tests/test_authorization_enforcement.py @@ -62,7 +62,9 @@ def test_policy_changes_selected_backend_execution() -> None: obligation, registry, context=context, - config=RoutingConfig(prefer_deterministic=True, max_selected_backends=1, enforced_lanes=frozenset({"authorization"})), + config=RoutingConfig( + prefer_deterministic=True, max_selected_backends=1, enforced_lanes=frozenset({"authorization"}) + ), policy=det_policy, ) assert [item.backend for item in routing.selected] == ["authorization-deterministic"] @@ -101,7 +103,7 @@ def test_enforced_authorization_emits_v2_preview_fields() -> None: ) evidence = evidence_items[0] assert evidence.routing_enforced is True - assert evidence.schema_version == "ovk.evidence.v2" + assert evidence.schema_version == "ovk.evidence.v3" assert evidence.obligation_id assert evidence.routing_id assert evidence.selected_backends @@ -154,7 +156,9 @@ def test_authorization_disagreement_blocks() -> None: outcome = aggregate_fail_dominant_v1( obligation_id="obl", selected=[ - BackendSelection(backend="z3-native", reason="p", expected_guarantee="smt_refutation_search", required=True), + BackendSelection( + backend="z3-native", reason="p", expected_guarantee="smt_refutation_search", required=True + ), BackendSelection( backend="authorization-deterministic", reason="c", diff --git a/tests/test_authorization_input_schema.py b/tests/test_authorization_input_schema.py index c2ae770..a334cd4 100644 --- a/tests/test_authorization_input_schema.py +++ b/tests/test_authorization_input_schema.py @@ -23,14 +23,10 @@ def test_authorization_schema_accepts_valid_protected_fixture() -> None: def test_authorization_schema_rejects_missing_routes_fixture() -> None: - errors = list( - validator().iter_errors(load_json("examples/auth_regression/input_malformed_missing_routes.json")) - ) + errors = list(validator().iter_errors(load_json("examples/auth_regression/input_malformed_missing_routes.json"))) assert errors def test_authorization_schema_rejects_bad_witness_fixture() -> None: - errors = list( - validator().iter_errors(load_json("examples/auth_regression/input_malformed_bad_witness.json")) - ) + errors = list(validator().iter_errors(load_json("examples/auth_regression/input_malformed_bad_witness.json"))) assert errors diff --git a/tests/test_backends.py b/tests/test_backends.py index 7e9bbb1..ccc28bc 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -8,8 +8,12 @@ def test_cedar_backend_pass_and_fail() -> None: - passed = evaluate_cedar_policy(json.loads(Path("examples/backends/cedar_pass.json").read_text(encoding="utf-8")), repo="r", head_sha="sha") - failed = evaluate_cedar_policy(json.loads(Path("examples/backends/cedar_fail.json").read_text(encoding="utf-8")), repo="r", head_sha="sha") + passed = evaluate_cedar_policy( + json.loads(Path("examples/backends/cedar_pass.json").read_text(encoding="utf-8")), repo="r", head_sha="sha" + ) + failed = evaluate_cedar_policy( + json.loads(Path("examples/backends/cedar_fail.json").read_text(encoding="utf-8")), repo="r", head_sha="sha" + ) assert passed.backend_claims[0].status.value == "pass" assert failed.backend_claims[0].status.value == "fail" diff --git a/tests/test_bench_badge.py b/tests/test_bench_badge.py index 6db7382..149df09 100644 --- a/tests/test_bench_badge.py +++ b/tests/test_bench_badge.py @@ -18,12 +18,17 @@ def test_render_badge_shape() -> None: "summary": {"cases_total": 100, "cases_passed": 100}, "timing_ms": {"p50": 1.0, "p95": 2.0, "max": 3.0}, } - badge = render_badge(leaderboard, verified_source_sha="abc1234deadbeef") + badge = render_badge( + leaderboard, + benchmark_source_sha="abc1234deadbeef", + verified_source_sha=None, + ) assert badge["schemaVersion"] == 1 assert badge["label"] == "FormalPR-Bench" assert "100/100" in badge["message"] assert badge["color"] == "brightgreen" - assert badge["verified_source_sha"] == "abc1234deadbeef" + assert badge["benchmark_source_sha"] == "abc1234deadbeef" + assert "verified_source_sha" not in badge def test_render_summary_includes_dimensions() -> None: @@ -37,12 +42,18 @@ def test_render_summary_includes_dimensions() -> None: }, "timing_ms": {"p50": 1.0, "p95": 2.0, "max": 3.0}, } - summary = render_summary(leaderboard, verified_source_sha="sha-source") + summary = render_summary( + leaderboard, + benchmark_source_sha="sha-bench", + verified_source_sha="sha-verified", + ) assert summary["schema_version"] == "formal_pr_bench.summary.v1" assert summary["cases_passed"] == 9 assert summary["timing_ms"]["p95"] == 2.0 assert "lane" in summary["by_category"] - assert summary["verified_source_sha"] == "sha-source" + assert summary["benchmark_source_sha"] == "sha-bench" + assert summary["verified_source_sha"] == "sha-verified" + assert "benchmark_source_sha" in summary["provenance_note"] assert "skip ci" in summary["provenance_note"] @@ -52,5 +63,7 @@ def test_render_summary_includes_real_diff_recall() -> None: "summary": {"cases_total": 10, "cases_passed": 9, "real_diff_recall": 0.95}, "timing_ms": {"p50": 1.0, "p95": 2.0, "max": 3.0}, } - summary = render_summary(leaderboard, verified_source_sha="x" * 40) + summary = render_summary(leaderboard, benchmark_source_sha="x" * 40) assert summary["real_diff_recall"] == 0.95 + assert summary["benchmark_source_sha"] == "x" * 40 + assert "verified_source_sha" not in summary diff --git a/tests/test_cache_worker_control_plane.py b/tests/test_cache_worker_control_plane.py index 187b7f9..6ef0510 100644 --- a/tests/test_cache_worker_control_plane.py +++ b/tests/test_cache_worker_control_plane.py @@ -78,12 +78,8 @@ def test_control_plane_cache_hit_and_subject_mismatch(tmp_path: Path) -> None: def test_policy_digest_mismatch_does_not_reuse_cache(tmp_path: Path) -> None: data = json.loads(Path("examples/auth_regression/input_admin_protected.json").read_text(encoding="utf-8")) registry = build_authorization_registry() - obligation_a = compile_authorization_obligation( - data, repo="r", head_sha="h", policy_digest="policy-a" - ) - obligation_b = compile_authorization_obligation( - data, repo="r", head_sha="h", policy_digest="policy-b" - ) + obligation_a = compile_authorization_obligation(data, repo="r", head_sha="h", policy_digest="policy-a") + obligation_b = compile_authorization_obligation(data, repo="r", head_sha="h", policy_digest="policy-b") assert obligation_a.policy_digest != obligation_b.policy_digest budget = _budget() cache = ControlPlaneResultCache(HardenedResultCache(tmp_path)) @@ -127,13 +123,20 @@ def test_worker_timeout_never_deterministic_pass(tmp_path: Path) -> None: def test_worker_env_allowlist_no_secret_inheritance(tmp_path: Path, monkeypatch) -> None: monkeypatch.setenv("GITHUB_TOKEN", "leak-me") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "also-leak") + monkeypatch.setenv("CUSTOM_UNKNOWN_CRED", "should-not-inherit") monkeypatch.setenv("PATH", os.environ.get("PATH", "")) worker = LocalSubprocessWorker(bound_roots=(tmp_path,)) result = worker.run( [ "python", "-c", - "import os; print(os.environ.get('GITHUB_TOKEN','')+os.environ.get('AWS_SECRET_ACCESS_KEY',''))", + ( + "import os; " + "print('TOK=' + os.environ.get('GITHUB_TOKEN','')); " + "print('AWS=' + os.environ.get('AWS_SECRET_ACCESS_KEY','')); " + "print('UNK=' + os.environ.get('CUSTOM_UNKNOWN_CRED','')); " + "print('SAFE=' + os.environ.get('OVK_WORKER_SAFE',''))" + ), ], cwd=tmp_path, env={"GITHUB_TOKEN": "should-not-pass", "OVK_WORKER_SAFE": "ok"}, @@ -143,6 +146,21 @@ def test_worker_env_allowlist_no_secret_inheritance(tmp_path: Path, monkeypatch) assert "leak-me" not in result.stdout assert "also-leak" not in result.stdout assert "should-not-pass" not in result.stdout + assert "should-not-inherit" not in result.stdout + assert "SAFE=ok" in result.stdout + + +def test_worker_rejects_non_positive_wall_budget(tmp_path: Path) -> None: + worker = LocalSubprocessWorker(bound_roots=(tmp_path,)) + result = worker.run( + ["python", "-c", "print('should-not-run')"], + cwd=tmp_path, + timeout_seconds=0, + ) + assert result.exit_code is None + assert result.timed_out is False + assert "non-positive" in result.stderr + assert "should-not-run" not in result.stdout def test_opa_runner_uses_worker_timeout(tmp_path: Path) -> None: diff --git a/tests/test_change_detection.py b/tests/test_change_detection.py index 5392f8c..9d8c730 100644 --- a/tests/test_change_detection.py +++ b/tests/test_change_detection.py @@ -17,10 +17,12 @@ def test_infrastructure_change_selects_infra_intent() -> None: def test_detect_change_surfaces_groups_files() -> None: - surfaces = detect_change_surfaces([ - ".github/workflows/verify.yml", - "src/middleware/auth.py", - "main.tf", - ]) + surfaces = detect_change_surfaces( + [ + ".github/workflows/verify.yml", + "src/middleware/auth.py", + "main.tf", + ] + ) domains = {surface.domain for surface in surfaces} assert {"ci_cd", "authorization", "infrastructure"}.issubset(domains) diff --git a/tests/test_compiler_runtime_wiring.py b/tests/test_compiler_runtime_wiring.py index da5f705..feca7f0 100644 --- a/tests/test_compiler_runtime_wiring.py +++ b/tests/test_compiler_runtime_wiring.py @@ -218,7 +218,9 @@ def test_deployment_github_environments_compiler() -> None: obligation, registry, context=ExecutionContext(subject=obligation.subject, budget=budget, policy_digest="p"), - config=RoutingConfig(prefer_deterministic=True, max_selected_backends=1, enforced_lanes=frozenset({"deployment"})), + config=RoutingConfig( + prefer_deterministic=True, max_selected_backends=1, enforced_lanes=frozenset({"deployment"}) + ), ) record = BackendControlPlane().execute(obligation, routing, registry=registry) assert record.results diff --git a/tests/test_decision.py b/tests/test_decision.py index db8fb8e..f5e1650 100644 --- a/tests/test_decision.py +++ b/tests/test_decision.py @@ -38,10 +38,7 @@ def test_unknown_requires_human_review_in_enforce_mode() -> None: def test_unknown_blocks_when_default_on_unknown_is_block() -> None: - assert ( - decide(make_bundle("unknown"), enforce=True, default_on_unknown="block") - == MergeRecommendation.BLOCK - ) + assert decide(make_bundle("unknown"), enforce=True, default_on_unknown="block") == MergeRecommendation.BLOCK def test_unknown_allows_with_warning_when_configured() -> None: diff --git a/tests/test_deterministic_worker_r2_pr7.py b/tests/test_deterministic_worker_r2_pr7.py new file mode 100644 index 0000000..cf9a14b --- /dev/null +++ b/tests/test_deterministic_worker_r2_pr7.py @@ -0,0 +1,107 @@ +"""PR7 — isolated deterministic worker tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + + +from ovk.adapters.authorization.deterministic_adapter import AuthorizationDeterministicAdapter +from ovk.core.backend_control_plane import BackendControlPlane +from ovk.core.deterministic_evaluators import evaluate_deterministic +from ovk.core.execution_budget import LocalSubprocessWorker +from ovk.core.execution_models import ExecutionBudget +from ovk.core.worker_runner import run_evaluator_in_worker + + +def test_deterministic_adapter_requires_worker() -> None: + adapter = AuthorizationDeterministicAdapter() + obligation = adapter.compile( + __import__( + "ovk.core.authorization_compiler", fromlist=["compile_authorization_obligation"] + ).compile_authorization_obligation( + json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")), + repo="r", + head_sha="h", + ), + __import__("ovk.core.routing_pipeline", fromlist=["route_compiled_obligation"]).route_compiled_obligation( + __import__( + "ovk.core.authorization_compiler", fromlist=["compile_authorization_obligation"] + ).compile_authorization_obligation( + json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")), + repo="r", + head_sha="h", + ), + lane="authorization", + ), + ) + budget = ExecutionBudget( + total_wall_time_seconds=30, + per_backend_wall_time_seconds=30, + max_memory_mb=512, + max_parallel_backends=1, + allow_network=False, + allow_repository_write=False, + ) + raw = adapter.run(obligation, budget, worker=None) + assert raw.termination == "tool_error" + assert raw.stderr and ("BackendWorker" in raw.stderr or "missing worker" in raw.stderr) + + +def test_deterministic_worker_runs_in_subprocess() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + worker = LocalSubprocessWorker() + outcome = run_evaluator_in_worker( + worker, + evaluator_id="authorization-deterministic", + payload={"input": data, "mode": "deterministic"}, + timeout_seconds=30.0, + ) + assert not outcome.timed_out + assert not outcome.worker_rejected + assert outcome.raw_result["status"] == "fail" + + +def test_zero_budget_rejected_before_worker_spawn() -> None: + worker = LocalSubprocessWorker() + outcome = run_evaluator_in_worker( + worker, + evaluator_id="authorization-deterministic", + payload={"input": {}}, + timeout_seconds=0.0, + ) + assert outcome.termination == "timeout" + assert outcome.worker_rejected + + +def test_control_plane_passes_worker_to_deterministic_adapter() -> None: + from ovk.adapters.authorization import build_authorization_registry + from ovk.core.authorization_compiler import compile_authorization_obligation + from ovk.core.routing_pipeline import route_compiled_obligation + + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + obligation = compile_authorization_obligation(data, repo="r", head_sha="h") + routing = route_compiled_obligation( + obligation, + lane="authorization", + policy={"budget": {"allowed_backends": ["authorization-deterministic"]}}, + ) + record = BackendControlPlane(worker=LocalSubprocessWorker()).execute( + obligation, + routing, + registry=build_authorization_registry(), + ) + assert record.results + assert record.results[0].status.value == "fail" + + +def test_evaluator_registry_covers_all_deterministic_backends() -> None: + for evaluator_id in ( + "authorization-deterministic", + "self-protection-deterministic", + "infrastructure-deterministic", + "ci-secrets-deterministic", + "deployment-deterministic", + ): + result = evaluate_deterministic(evaluator_id, {"input": {}}) + assert "raw_result" in result diff --git a/tests/test_emit_github_check.py b/tests/test_emit_github_check.py index 9e35903..3be9f8a 100644 --- a/tests/test_emit_github_check.py +++ b/tests/test_emit_github_check.py @@ -50,17 +50,20 @@ def test_emit_github_check_posts_check_run(tmp_path: Path, monkeypatch) -> None: response.__enter__ = MagicMock(return_value=response) response.__exit__ = MagicMock(return_value=False) - with patch("sys.argv", [ - "emit_github_check.py", - "--evidence", - str(evidence), - "--markdown", - str(markdown), - "--repo", - "owner/repo", - "--head-sha", - "abc123", - ]): + with patch( + "sys.argv", + [ + "emit_github_check.py", + "--evidence", + str(evidence), + "--markdown", + str(markdown), + "--repo", + "owner/repo", + "--head-sha", + "abc123", + ], + ): with patch("urllib.request.urlopen", return_value=response) as urlopen: assert main() == 0 request = urlopen.call_args.args[0] @@ -75,15 +78,18 @@ def test_emit_github_check_api_failure_returns_one(tmp_path: Path, monkeypatch) _write_evidence(evidence) monkeypatch.setenv("GITHUB_TOKEN", "test-token") - with patch("sys.argv", [ - "emit_github_check.py", - "--evidence", - str(evidence), - "--repo", - "owner/repo", - "--head-sha", - "abc123", - ]): + with patch( + "sys.argv", + [ + "emit_github_check.py", + "--evidence", + str(evidence), + "--repo", + "owner/repo", + "--head-sha", + "abc123", + ], + ): with patch("scripts.emit_github_check._post_check_run", return_value=False): assert main() == 1 diff --git a/tests/test_evidence_v3_r2_pr9.py b/tests/test_evidence_v3_r2_pr9.py new file mode 100644 index 0000000..f753e1f --- /dev/null +++ b/tests/test_evidence_v3_r2_pr9.py @@ -0,0 +1,108 @@ +"""PR9 — evidence v3 and material-set binding tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from ovk.core.adapter_runtime import execute_obligations +from ovk.core.attestation import bundle_to_statement +from ovk.core.attestation_binding import verify_bundle_statement_binding +from ovk.core.bundle import make_bundle +from ovk.core.evidence_invariants import check_evidence_bundle_invariants +from ovk.core.materials import compute_material_set_digest +from ovk.core.provenance import build_provenance_statement +from ovk.core.schema_validation import load_json, validate_against_schema +from ovk.paths import schema_path + + +def _auth_policy(): + return { + "routing": { + "enforced_lanes": ["authorization"], + "prefer_deterministic": True, + }, + "budget": {"allowed_backends": ["authorization-deterministic"]}, + } + + +def test_compute_material_set_digest_is_order_insensitive() -> None: + materials_a = [ + {"material_id": "b", "sha256": "2" * 64}, + {"material_id": "a", "sha256": "1" * 64}, + ] + materials_b = list(reversed(materials_a)) + assert compute_material_set_digest(materials_a) == compute_material_set_digest(materials_b) + + +def test_enforced_emission_uses_evidence_v3_with_material_set_digest() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + evidence_items = execute_obligations( + [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}], + {}, + repo="example/repo", + head_sha="abc", + use_cache=False, + policy=_auth_policy(), + evidence_schema_version="ovk.evidence.v3", + ) + evidence = evidence_items[0] + assert evidence.schema_version == "ovk.evidence.v3" + assert evidence.material_set_digest + assert evidence.material_set_digest == compute_material_set_digest(evidence.materials) + + +def test_evidence_v3_schema_validation() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + evidence = execute_obligations( + [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}], + {}, + repo="example/repo", + head_sha="abc", + use_cache=False, + policy=_auth_policy(), + evidence_schema_version="ovk.evidence.v3", + )[0] + schema = load_json(schema_path("verification.evidence.v3.schema.json")) + report = validate_against_schema(evidence.model_dump(mode="json"), schema) + assert report.valid, [issue.message for issue in report.issues] + + +def test_v3_invariants_and_attestation_material_binding() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + evidence = execute_obligations( + [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}], + {}, + repo="example/repo", + head_sha="abc", + use_cache=False, + policy=_auth_policy(), + evidence_schema_version="ovk.evidence.v3", + )[0] + bundle = make_bundle([evidence]) + issues = check_evidence_bundle_invariants(bundle) + assert not [issue for issue in issues if issue.severity == "error"] + + statement = bundle_to_statement(bundle) + binding_issues = verify_bundle_statement_binding(bundle, statement) + assert not binding_issues + + provenance = build_provenance_statement(bundle) + assert evidence.material_set_digest in provenance["control_plane"]["material_set_digests"] + + +def test_adversarial_material_set_digest_mismatch_fails_closed() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + evidence = execute_obligations( + [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}], + {}, + repo="example/repo", + head_sha="abc", + use_cache=False, + policy=_auth_policy(), + evidence_schema_version="ovk.evidence.v3", + )[0] + tampered = evidence.model_copy(update={"material_set_digest": "deadbeef"}) + bundle = make_bundle([tampered]) + issues = check_evidence_bundle_invariants(bundle) + assert any("material_set_digest" in issue.message for issue in issues) diff --git a/tests/test_execution_models.py b/tests/test_execution_models.py index 08e26ff..f4f23ba 100644 --- a/tests/test_execution_models.py +++ b/tests/test_execution_models.py @@ -367,12 +367,30 @@ def test_attempt_id_excludes_wall_clock_timestamps() -> None: assert "attempt_id" not in digest_input assert "started_at" not in digest_input assert "finished_at" not in digest_input + assert "duration_ms" not in digest_input shifted = attempt.model_copy( - update={"started_at": "2099-01-01T00:00:00Z", "finished_at": "2099-01-01T00:00:01Z"} + update={ + "started_at": "2099-01-01T00:00:00Z", + "finished_at": "2099-01-01T00:00:01Z", + "duration_ms": 99999.0, + } ) assert compute_attempt_id(shifted) == compute_attempt_id(attempt) +def test_attempt_id_stable_across_duration_jitter() -> None: + """Equivalent semantic attempts must share IDs despite timing jitter.""" + base = _attempt("bo-stable") + variants = [ + base.model_copy(update={"duration_ms": 1.0}), + base.model_copy(update={"duration_ms": 50.5}), + base.model_copy(update={"duration_ms": 1200.0, "started_at": "2026-07-01T00:00:00Z"}), + base.model_copy(update={"duration_ms": 0.0}), + ] + ids = {compute_attempt_id(item) for item in [base, *variants]} + assert len(ids) == 1 + + def test_attempt_id_changes_when_termination_changes() -> None: attempt = _attempt("bo-1") other = attempt.model_copy(update={"termination": "timeout", "exit_code": None}) diff --git a/tests/test_formalpr_holdout_runner.py b/tests/test_formalpr_holdout_runner.py index 147828a..8058431 100644 --- a/tests/test_formalpr_holdout_runner.py +++ b/tests/test_formalpr_holdout_runner.py @@ -2,12 +2,21 @@ from __future__ import annotations +import hashlib +import io import json +import tarfile from pathlib import Path import pytest -from scripts.run_formalpr_holdout import assert_aggregate_safe +from scripts.run_formalpr_holdout import ( + assert_aggregate_safe, + extract_tarball, + validate_aggregate_schema, + verify_asset_sha256, + _isolated_eval_env, +) def _valid_aggregate() -> dict: @@ -49,6 +58,10 @@ def test_assert_aggregate_safe_accepts_valid_payload() -> None: assert_aggregate_safe(_valid_aggregate()) +def test_validate_aggregate_schema_accepts_valid_payload() -> None: + validate_aggregate_schema(_valid_aggregate()) + + def test_assert_aggregate_safe_rejects_case_id_leak() -> None: payload = _valid_aggregate() payload["notes"] = "syn-auth-bypass-01" @@ -63,6 +76,107 @@ def test_assert_aggregate_safe_rejects_label_flag() -> None: assert_aggregate_safe(payload) +def test_validate_aggregate_schema_rejects_missing_lane_metric() -> None: + payload = _valid_aggregate() + del payload["lanes"]["authorization"]["precision"] + with pytest.raises(SystemExit, match="fail-closed"): + validate_aggregate_schema(payload) + + +def test_verify_asset_sha256_accepts_and_rejects(tmp_path: Path) -> None: + asset = tmp_path / "asset.tar.gz" + asset.write_bytes(b"holdout-bytes") + digest = hashlib.sha256(asset.read_bytes()).hexdigest() + assert verify_asset_sha256(asset, digest) == digest + with pytest.raises(SystemExit, match="SHA-256 mismatch"): + verify_asset_sha256(asset, "0" * 64) + with pytest.raises(SystemExit, match="64-character"): + verify_asset_sha256(asset, "not-a-digest") + + +def test_extract_tarball_rejects_symlink(tmp_path: Path) -> None: + tarball = tmp_path / "bad.tar.gz" + with tarfile.open(tarball, "w:gz") as tar: + info = tarfile.TarInfo(name="root/link") + info.type = tarfile.SYMTYPE + info.linkname = "../outside" + tar.addfile(info) + with pytest.raises(SystemExit, match="link member forbidden"): + extract_tarball(tarball, tmp_path / "out") + + +def test_extract_tarball_rejects_traversal(tmp_path: Path) -> None: + tarball = tmp_path / "trav.tar.gz" + with tarfile.open(tarball, "w:gz") as tar: + data = b"x" + info = tarfile.TarInfo(name="../evil.txt") + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + with pytest.raises(SystemExit, match="unsafe path|path traversal"): + extract_tarball(tarball, tmp_path / "out") + + +def test_extract_tarball_accepts_regular_tree(tmp_path: Path) -> None: + tarball = tmp_path / "good.tar.gz" + with tarfile.open(tarball, "w:gz") as tar: + dir_info = tarfile.TarInfo(name="release-root") + dir_info.type = tarfile.DIRTYPE + tar.addfile(dir_info) + payload = b'{"ok": true}' + file_info = tarfile.TarInfo(name="release-root/readme.json") + file_info.size = len(payload) + tar.addfile(file_info, io.BytesIO(payload)) + root = extract_tarball(tarball, tmp_path / "out") + assert root.name == "release-root" + assert (root / "readme.json").read_bytes() == payload + + +def test_isolated_eval_env_strips_tokens(monkeypatch) -> None: + monkeypatch.setenv("HOLDOUT_DOWNLOAD_TOKEN", "secret-holdout") + monkeypatch.setenv("GITHUB_TOKEN", "secret-gh") + monkeypatch.setenv("PATH", "/usr/bin") + env = _isolated_eval_env() + assert "HOLDOUT_DOWNLOAD_TOKEN" not in env + assert "GITHUB_TOKEN" not in env + assert env.get("PATH") == "/usr/bin" + + +def test_predictions_digest_rejects_embedded_labels(tmp_path: Path) -> None: + from scripts.digest_holdout_predictions import assert_predictions_label_free, digest_predictions_file + + clean = {"predictions": [{"case_id": "case-1", "status": "fail", "merge_recommendation": "block"}]} + assert_predictions_label_free(clean) + path = tmp_path / "predictions.json" + path.write_text(json.dumps(clean), encoding="utf-8") + record = digest_predictions_file(path) + assert record["label_free"] is True + assert len(record["sha256"]) == 64 + + dirty = { + "predictions": [ + { + "case_id": "case-1", + "status": "fail", + "expected_status": "fail", + } + ] + } + with pytest.raises(SystemExit, match="fail-closed"): + assert_predictions_label_free(dirty) + + +def test_predictions_digest_rejects_forbidden_substring(tmp_path: Path) -> None: + from scripts.digest_holdout_predictions import digest_predictions_file + + path = tmp_path / "predictions.json" + path.write_text( + json.dumps({"predictions": [{"case_id": "x", "note": "ground_truth_class leaked"}]}), + encoding="utf-8", + ) + with pytest.raises(SystemExit, match="fail-closed"): + digest_predictions_file(path) + + def test_run_formalpr_holdout_against_local_artifact(tmp_path: Path) -> None: """End-to-end against a sibling-packaged release if present.""" holdout_root = Path(__file__).resolve().parents[1].parent / "FormalPR-Holdout" @@ -99,10 +213,13 @@ def test_run_formalpr_holdout_against_local_artifact(tmp_path: Path) -> None: pred_path = tmp_path / "predictions.json" pred_path.write_text(json.dumps(preds), encoding="utf-8") out = tmp_path / "agg.json" + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() rc = runner.main( [ "--artifact", str(artifact), + "--asset-sha256", + digest, "--tag", "v0.1.0-synthetic", "--predictions", diff --git a/tests/test_infra_graph.py b/tests/test_infra_graph.py index 3a978e8..2c20dca 100644 --- a/tests/test_infra_graph.py +++ b/tests/test_infra_graph.py @@ -29,6 +29,8 @@ def test_graph_disconnected_sensitive_resource_allows() -> None: def test_empty_graph_becomes_invalid_infra_input() -> None: - evidence = evaluate_infra_exposure(graph_to_infra_input({"nodes": [], "edges": []}), repo="example/repo", head_sha="abc") + evidence = evaluate_infra_exposure( + graph_to_infra_input({"nodes": [], "edges": []}), repo="example/repo", head_sha="abc" + ) assert evidence.backend_claims[0].status.value == "unknown" assert evidence.decision["merge_recommendation"] == "require_human_review" diff --git a/tests/test_ingest_external_pilot_metrics.py b/tests/test_ingest_external_pilot_metrics.py index dec18fb..0544086 100644 --- a/tests/test_ingest_external_pilot_metrics.py +++ b/tests/test_ingest_external_pilot_metrics.py @@ -141,7 +141,15 @@ def test_validate_registry_rejects_invalid_row(tmp_path: Path) -> None: registry = { "schema_version": "ovk.external_pilots_registry.v1", "updated_at": "2026-06-10T00:00:00Z", - "external_pilots": [{"repository": "bad/repo", "status": "unknown", "check_types": ["ci_secrets"], "strict_enabled": False, "ovk_version_pin": "1.2.0"}], + "external_pilots": [ + { + "repository": "bad/repo", + "status": "unknown", + "check_types": ["ci_secrets"], + "strict_enabled": False, + "ovk_version_pin": "1.2.0", + } + ], } with pytest.raises(ValueError, match="external pilots registry failed schema validation"): validate_registry(registry) diff --git a/tests/test_material_reference_sizes.py b/tests/test_material_reference_sizes.py index a1efea8..127ca81 100644 --- a/tests/test_material_reference_sizes.py +++ b/tests/test_material_reference_sizes.py @@ -1,6 +1,8 @@ from ovk.core.ci_secrets_compiler import compile_ci_secrets_obligation +from ovk.core.compiler_bridge import material_refs_from_digest from ovk.core.materials import canonical_material_bytes, material_reference_from_payload from ovk.core.self_protection_compiler import compile_self_protection_obligation +from ovk.core.shadow_obligation import build_shadow_obligation def test_material_reference_size_matches_canonical_payload_bytes() -> None: @@ -36,6 +38,43 @@ def test_self_protection_material_sizes_bind_each_payload() -> None: assert by_id["self-protection-input"].size_bytes == len(canonical_material_bytes(data)) +def test_ci_secrets_material_size_binds_payload_not_digest() -> None: + data = { + "workflows": { + "ci.yml": { + "on": "pull_request_target", + "jobs": {"build": {"runs-on": "ubuntu-latest", "steps": []}}, + } + } + } + obligation = compile_ci_secrets_obligation(data, repo="example/repo", head_sha="head") + material = obligation.materials[0] + assert material.size_bytes == len(canonical_material_bytes(data)) + assert material.size_bytes != len(material.sha256) + + +def test_shadow_and_bridge_material_sizes_bind_payload() -> None: + data = {"routes": [{"path": "/admin", "methods": ["GET"]}]} + shadow = build_shadow_obligation( + lane="authorization", + data=data, + repo="example/repo", + head_sha="head", + base_sha="base", + intent_id="admin-only-routes", + ) + assert shadow.materials[0].size_bytes == len(canonical_material_bytes(data)) + bridged = material_refs_from_digest( + material_id="bridge-material", + kind="diff", + uri="ovk-material:bridge", + payload=data, + source_revision="head", + ) + assert bridged.size_bytes == len(canonical_material_bytes(data)) + assert bridged.size_bytes != len(bridged.sha256) + + def test_ci_secrets_legacy_material_size_binds_input_payload() -> None: data = { "trust_context": "untrusted_fork_pr", diff --git a/tests/test_native_backends.py b/tests/test_native_backends.py index 39394b8..974e3e2 100644 --- a/tests/test_native_backends.py +++ b/tests/test_native_backends.py @@ -20,10 +20,7 @@ def _assert_backend(backend: str) -> None: + (f", detail={result.detail}" if getattr(result, "detail", None) else "") ) if result.binary_present and backend in TIER1_NATIVE_EXECUTION_BACKENDS: - assert result.used_native_binary, ( - f"{backend} detected {result.binary_name} " - "but did not report native use" - ) + assert result.used_native_binary, f"{backend} detected {result.binary_name} but did not report native use" @requires_native_ci diff --git a/tests/test_opa_infra_domain.py b/tests/test_opa_infra_domain.py index 12fe204..5284420 100644 --- a/tests/test_opa_infra_domain.py +++ b/tests/test_opa_infra_domain.py @@ -5,14 +5,18 @@ def test_opa_infra_domain_pack_blocks_public_sensitive_resource() -> None: - payload = json.loads(Path("examples/infrastructure_exposure/input_public_sensitive_resource.json").read_text(encoding="utf-8")) + payload = json.loads( + Path("examples/infrastructure_exposure/input_public_sensitive_resource.json").read_text(encoding="utf-8") + ) evidence = evaluate_infra_exposure_opa(payload, repo="r", head_sha="sha") assert evidence.backend_claims[0].status.value == "fail" assert evidence.counterexamples[0]["failure_mode"] == "sensitive_resource_publicly_exposed" def test_opa_infra_domain_pack_allows_private_resource() -> None: - payload = json.loads(Path("examples/infrastructure_exposure/input_private_sensitive_resource.json").read_text(encoding="utf-8")) + payload = json.loads( + Path("examples/infrastructure_exposure/input_private_sensitive_resource.json").read_text(encoding="utf-8") + ) evidence = evaluate_infra_exposure_opa(payload, repo="r", head_sha="sha") assert evidence.backend_claims[0].status.value == "pass" diff --git a/tests/test_phase3_trust.py b/tests/test_phase3_trust.py index 0d9d9ac..0450feb 100644 --- a/tests/test_phase3_trust.py +++ b/tests/test_phase3_trust.py @@ -12,7 +12,9 @@ def test_adversarial_sha_mismatch_fails_quality_gate() -> None: - bundle = EvidenceBundle.model_validate(read_json_file(Path("examples/evidence_quality/adversarial_sha_mismatch.json"))) + bundle = EvidenceBundle.model_validate( + read_json_file(Path("examples/evidence_quality/adversarial_sha_mismatch.json")) + ) report = build_evidence_quality_report(bundle) assert report.passed is False messages = {issue.message for issue in report.issues} diff --git a/tests/test_phase7_v1.py b/tests/test_phase7_v1.py index 3ff427f..4a71ee4 100644 --- a/tests/test_phase7_v1.py +++ b/tests/test_phase7_v1.py @@ -50,8 +50,7 @@ def test_bench_cli_writes_leaderboard(tmp_path: Path) -> None: def test_v1_readiness_checklist() -> None: metadata = release_metadata() backends = { - manifest.get("tool", {}).get("name") - for manifest in CapabilityRegistry.from_directory(Path("adapters")).all() + manifest.get("tool", {}).get("name") for manifest in CapabilityRegistry.from_directory(Path("adapters")).all() } required_backends = {"opa", "z3", "cedar", "tla+", "kani", "dafny", "verus", "lean", "cbmc", "alloy"} assert required_backends.issubset(backends) diff --git a/tests/test_policy_loading_fail_closed.py b/tests/test_policy_loading_fail_closed.py index 96f3865..e1f2705 100644 --- a/tests/test_policy_loading_fail_closed.py +++ b/tests/test_policy_loading_fail_closed.py @@ -15,9 +15,7 @@ def test_malformed_policy_yaml_is_rejected(tmp_path: Path) -> None: def test_schema_invalid_policy_is_rejected(tmp_path: Path) -> None: config = tmp_path / "config.yml" config.write_text( - "schema_version: ovk.config.v1\n" - "mode: strict\n" - "default_on_unknown: allow\n", + "schema_version: ovk.config.v1\nmode: strict\ndefault_on_unknown: allow\n", encoding="utf-8", ) with pytest.raises(ValueError, match="failed schema validation"): diff --git a/tests/test_remaining_lane_enforcement.py b/tests/test_remaining_lane_enforcement.py index 556e31e..c0c26e0 100644 --- a/tests/test_remaining_lane_enforcement.py +++ b/tests/test_remaining_lane_enforcement.py @@ -254,7 +254,7 @@ def test_enforced_infrastructure_blocks_public_sensitive() -> None: ) evidence = evidence_items[0] assert evidence.routing_enforced is True - assert evidence.schema_version == "ovk.evidence.v2" + assert evidence.schema_version == "ovk.evidence.v3" assert evidence.decision.get("merge_recommendation") == "block" assert "infrastructure-deterministic" in (evidence.selected_backends or []) diff --git a/tests/test_render_bundle.py b/tests/test_render_bundle.py index f446690..dd161cd 100644 --- a/tests/test_render_bundle.py +++ b/tests/test_render_bundle.py @@ -7,18 +7,14 @@ def test_bundle_blocks_when_evidence_fails() -> None: - data = json.loads( - Path("examples/no_agent_self_approval/input_gate_removed.json").read_text(encoding="utf-8") - ) + data = json.loads(Path("examples/no_agent_self_approval/input_gate_removed.json").read_text(encoding="utf-8")) evidence = evaluate_self_protection(data, repo="example/repo", head_sha="abc") bundle = make_bundle([evidence]) assert bundle.decision["merge_recommendation"] == "block" def test_renderer_includes_guarantee_status_and_counterexample() -> None: - data = json.loads( - Path("examples/no_agent_self_approval/input_gate_removed.json").read_text(encoding="utf-8") - ) + data = json.loads(Path("examples/no_agent_self_approval/input_gate_removed.json").read_text(encoding="utf-8")) evidence = evaluate_self_protection(data, repo="example/repo", head_sha="abc") bundle = make_bundle([evidence]) rendered = render_bundle_markdown(bundle) diff --git a/tests/test_render_pilot_metrics.py b/tests/test_render_pilot_metrics.py index 185c970..124b87c 100644 --- a/tests/test_render_pilot_metrics.py +++ b/tests/test_render_pilot_metrics.py @@ -100,8 +100,26 @@ def test_render_merge_preserves_registry_on_rerender(tmp_path: Path) -> None: def test_merge_external_pilots_registry_wins_on_conflict() -> None: merged = merge_external_pilots( - [{"repository": "org/repo", "status": "advisory", "check_types": ["ci_secrets"], "strict_enabled": False, "ovk_version_pin": "1.2.0", "prs_evaluated": 5}], - [{"repository": "org/repo", "status": "recruiting", "check_types": ["ci_secrets"], "strict_enabled": False, "ovk_version_pin": "1.2.0", "prs_evaluated": 1}], + [ + { + "repository": "org/repo", + "status": "advisory", + "check_types": ["ci_secrets"], + "strict_enabled": False, + "ovk_version_pin": "1.2.0", + "prs_evaluated": 5, + } + ], + [ + { + "repository": "org/repo", + "status": "recruiting", + "check_types": ["ci_secrets"], + "strict_enabled": False, + "ovk_version_pin": "1.2.0", + "prs_evaluated": 1, + } + ], ) assert len(merged) == 1 assert merged[0]["status"] == "advisory" diff --git a/tests/test_repair_loop.py b/tests/test_repair_loop.py index 31eb478..bc1b46c 100644 --- a/tests/test_repair_loop.py +++ b/tests/test_repair_loop.py @@ -14,9 +14,7 @@ def test_ci_secrets_repair_loop_failing_diff_blocks() -> None: result = run_check(diff_text=diff_text, repo="test/repo", head_sha="fail", use_cache=False) assert result.bundle.decision.get("merge_recommendation") == "block" counterexamples = [ - counterexample - for evidence in result.bundle.evidence - for counterexample in evidence.counterexamples + counterexample for evidence in result.bundle.evidence for counterexample in evidence.counterexamples ] assert counterexamples diff --git a/tests/test_repo_memory_trust.py b/tests/test_repo_memory_trust.py index 4baf2ce..f5faf60 100644 --- a/tests/test_repo_memory_trust.py +++ b/tests/test_repo_memory_trust.py @@ -10,9 +10,7 @@ def _write_run(path: Path, status: str) -> None: { "backend_outcomes": [{"backend": "z3", "status": status}], "lanes": ["no-admin-route-bypass"], - "decision": { - "merge_recommendation": "block" if status == "fail" else "require_human_review" - }, + "decision": {"merge_recommendation": "block" if status == "fail" else "require_human_review"}, } ), encoding="utf-8", diff --git a/tests/test_result_cache_semantics.py b/tests/test_result_cache_semantics.py index 250c0cb..06f132b 100644 --- a/tests/test_result_cache_semantics.py +++ b/tests/test_result_cache_semantics.py @@ -33,8 +33,22 @@ def fake_evaluate_lane(lane, data, *, repo, head_sha, base_sha, input_format, po monkeypatch.setattr(adapter_runtime, "evaluate_lane", fake_evaluate_lane) raw_input = {"resource_changes": []} - native = [{"lane": "infrastructure", "intent_id": "no-public-sensitive-resource", "input": raw_input, "input_format": "infra"}] - terraform = [{"lane": "infrastructure", "intent_id": "no-public-sensitive-resource", "input": raw_input, "input_format": "terraform"}] + native = [ + { + "lane": "infrastructure", + "intent_id": "no-public-sensitive-resource", + "input": raw_input, + "input_format": "infra", + } + ] + terraform = [ + { + "lane": "infrastructure", + "intent_id": "no-public-sensitive-resource", + "input": raw_input, + "input_format": "terraform", + } + ] adapter_runtime.execute_obligations( native, diff --git a/tests/test_routing_pipeline_r2_pr6.py b/tests/test_routing_pipeline_r2_pr6.py new file mode 100644 index 0000000..8a0bbb7 --- /dev/null +++ b/tests/test_routing_pipeline_r2_pr6.py @@ -0,0 +1,138 @@ +"""PR6 — single authoritative routing pipeline tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from ovk.core.adapter_runtime import execute_obligations +from ovk.core.kernel import execute_kernel +from ovk.core.routing_pipeline import ( + build_authoritative_routing_plan, + route_compiled_obligation, + compile_typed_obligation, +) +from ovk.core.router import routing_decision_to_legacy_dict +from ovk.mcp_server import select_backends + + +def _auth_policy(**routing_overrides): + routing = { + "mode": "shadow", + "enforced_lanes": ["authorization"], + "max_selected_backends": 1, + "prefer_deterministic": True, + "allow_fallback": False, + } + routing.update(routing_overrides) + return {"routing": routing, "budget": {"allowed_backends": ["authorization-deterministic"]}} + + +def test_compile_before_route_produces_single_routing_id() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + obligations = [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}] + plan = build_authoritative_routing_plan( + obligations, + policy=_auth_policy(), + repo="example/repo", + head_sha="abc", + ) + assert len(plan.routing_by_intent) == 1 + routing = plan.routing_by_intent["no-admin-route-bypass"] + assert routing.routing_id + assert routing.obligation_id == plan.typed_obligations["no-admin-route-bypass"].obligation_id + + +def test_kernel_mcp_and_evidence_share_routing_id() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + policy = _auth_policy() + obligations = [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}] + plan = build_authoritative_routing_plan( + obligations, + policy=policy, + repo="example/repo", + head_sha="abc", + ) + expected_routing_id = plan.routing_by_intent["no-admin-route-bypass"].routing_id + + mcp_plan = select_backends( + "no-admin-route-bypass", + lane="authorization", + lane_input=data, + repo="example/repo", + head_sha="abc", + policy=policy, + ) + assert mcp_plan["routing_id"] == expected_routing_id + + evidence_items = execute_obligations( + obligations, + plan.legacy_routing_by_intent(), + repo="example/repo", + head_sha="abc", + use_cache=False, + policy=policy, + ) + assert evidence_items[0].routing_id == expected_routing_id + routing_artifacts = [ + artifact for artifact in evidence_items[0].generated_artifacts if artifact.get("kind") == "backend_routing" + ] + assert routing_artifacts + assert routing_artifacts[0]["routing_id"] == expected_routing_id + + +def test_route_intent_and_route_obligation_diverge_for_legacy_multi_select() -> None: + """Legacy manifest routing may differ; enforced typed path is authoritative.""" + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + obligation = compile_typed_obligation( + lane="authorization", + data=data, + repo="r", + head_sha="h", + policy=_auth_policy(max_selected_backends=1), + ) + typed = route_compiled_obligation(obligation, lane="authorization", policy=_auth_policy(max_selected_backends=1)) + assert len(typed.selected) == 1 + assert typed.selected[0].backend == "authorization-deterministic" + legacy_dict = routing_decision_to_legacy_dict(typed, intent_id=obligation.intent_id) + assert legacy_dict["routing_id"] == typed.routing_id + + +def test_kernel_uses_authoritative_routing_for_obligations() -> None: + diff_text = Path("examples/multi_surface/pr_combined.diff").read_text(encoding="utf-8") + from dataclasses import replace + + from ovk.core.context import build_repository_context + + ctx = build_repository_context( + changed_files=["src/routes/admin.ts", ".github/workflows/ci.yml", "infra/main.tf"], + repo="test/repo", + head_sha="deadbeef", + ) + ctx = replace( + ctx, + policy={ + "routing": { + "enforced_lanes": ["authorization"], + "prefer_deterministic": True, + }, + "budget": {"allowed_backends": ["authorization-deterministic"]}, + }, + ) + result = execute_kernel( + diff_text=diff_text, + use_cache=False, + repo="test/repo", + head_sha="deadbeef", + context=ctx, + ) + auth_evidence = next( + item for item in result.bundle.evidence if item.intent.get("intent_id") == "no-admin-route-bypass" + ) + assert auth_evidence.routing_id + auth_routing = next( + item + for item in result.routing + if item.get("obligation_id") == auth_evidence.obligation_id or item.get("intent_id") == "no-admin-route-bypass" + ) + assert auth_routing["routing_id"] == auth_evidence.routing_id diff --git a/tests/test_runtime_cache_regimes.py b/tests/test_runtime_cache_regimes.py index 20f860e..5573551 100644 --- a/tests/test_runtime_cache_regimes.py +++ b/tests/test_runtime_cache_regimes.py @@ -5,9 +5,7 @@ def _auth_input() -> dict: - return json.loads( - Path("examples/auth_regression/input_admin_protected.json").read_text(encoding="utf-8") - ) + return json.loads(Path("examples/auth_regression/input_admin_protected.json").read_text(encoding="utf-8")) def test_enforced_execution_does_not_reuse_legacy_flat_cache(tmp_path: Path) -> None: diff --git a/tests/test_scripts_sprint2.py b/tests/test_scripts_sprint2.py index 692e735..4ecf5e7 100644 --- a/tests/test_scripts_sprint2.py +++ b/tests/test_scripts_sprint2.py @@ -9,15 +9,7 @@ def test_normalize_required_checks_script(tmp_path: Path, monkeypatch) -> None: input_path = tmp_path / "branch_protection.json" output_path = tmp_path / "required_checks.json" input_path.write_text( - json.dumps( - { - "after_branch_protection": { - "required_status_checks": { - "contexts": ["unit-tests", "ovk-verify"] - } - } - } - ), + json.dumps({"after_branch_protection": {"required_status_checks": {"contexts": ["unit-tests", "ovk-verify"]}}}), encoding="utf-8", ) monkeypatch.setattr( diff --git a/tests/test_self_protection_enforcement.py b/tests/test_self_protection_enforcement.py index c0818c6..265d3ba 100644 --- a/tests/test_self_protection_enforcement.py +++ b/tests/test_self_protection_enforcement.py @@ -71,7 +71,7 @@ def test_enforced_self_protection_blocks_gate_removal() -> None: ) evidence = evidence_items[0] assert evidence.routing_enforced is True - assert evidence.schema_version == "ovk.evidence.v2" + assert evidence.schema_version == "ovk.evidence.v3" assert evidence.decision.get("merge_recommendation") == "block" assert evidence.selected_backends assert "self-protection-deterministic" in (evidence.selected_backends or []) or "opa-native" in ( @@ -80,13 +80,18 @@ def test_enforced_self_protection_blocks_gate_removal() -> None: def test_untrusted_metadata_cannot_allow(monkeypatch: pytest.MonkeyPatch) -> None: - data = _load_example("input_gate_preserved.json") if Path("examples/no_agent_self_approval/input_gate_preserved.json").exists() or Path("ovk/package_data/examples/no_agent_self_approval/input_gate_preserved.json").exists() else { - "actor": {"type": "ai_agent", "id": "bot"}, - "changed_files": ["README.md"], - "before": {"required_checks": ["ovk-verify"]}, - "after": {"required_checks": ["ovk-verify"]}, - "ovk_gate_name": "ovk-verify", - } + data = ( + _load_example("input_gate_preserved.json") + if Path("examples/no_agent_self_approval/input_gate_preserved.json").exists() + or Path("ovk/package_data/examples/no_agent_self_approval/input_gate_preserved.json").exists() + else { + "actor": {"type": "ai_agent", "id": "bot"}, + "changed_files": ["README.md"], + "before": {"required_checks": ["ovk-verify"]}, + "after": {"required_checks": ["ovk-verify"]}, + "ovk_gate_name": "ovk-verify", + } + ) if isinstance(data, str): data = _load_example("input_gate_preserved.json") evidence_items = execute_obligations( diff --git a/tests/test_self_protection_sprint1.py b/tests/test_self_protection_sprint1.py index 9257e07..9b32a4d 100644 --- a/tests/test_self_protection_sprint1.py +++ b/tests/test_self_protection_sprint1.py @@ -6,9 +6,7 @@ def test_missing_required_check_metadata_returns_unknown() -> None: - data = json.loads( - Path("examples/no_agent_self_approval/input_missing_metadata.json").read_text(encoding="utf-8") - ) + data = json.loads(Path("examples/no_agent_self_approval/input_missing_metadata.json").read_text(encoding="utf-8")) evidence = evaluate_self_protection(data, repo="example/repo", head_sha="abc") assert evidence.backend_claims[0].status.value == "unknown" assert evidence.decision["merge_recommendation"] == "require_human_review" diff --git a/tests/test_shadow_control_plane.py b/tests/test_shadow_control_plane.py index ad3e4fc..8411546 100644 --- a/tests/test_shadow_control_plane.py +++ b/tests/test_shadow_control_plane.py @@ -23,7 +23,9 @@ from ovk.core.shadow_obligation import build_shadow_obligation -def _result(backend: str, status: VerificationStatus, *, guarantee: str = "smt_refutation_search") -> NormalizedBackendResult: +def _result( + backend: str, status: VerificationStatus, *, guarantee: str = "smt_refutation_search" +) -> NormalizedBackendResult: return NormalizedBackendResult( attempt_id=f"att-{backend}", backend=backend, diff --git a/tests/test_sigstore_release.py b/tests/test_sigstore_release.py index 537ca1e..a9470f8 100644 --- a/tests/test_sigstore_release.py +++ b/tests/test_sigstore_release.py @@ -21,27 +21,21 @@ def test_github_certificate_identity_from_workflow_ref() -> None: identity = github_certificate_identity( - workflow_ref=( - "fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.0" - ) + workflow_ref=("fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.0") ) assert identity == ( - "https://github.com/fraware/open-verification-kernel/" - ".github/workflows/publish.yml@refs/tags/v1.2.0" + "https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.0" ) def test_github_certificate_identity_rejects_foreign_repo() -> None: with pytest.raises(ValueError, match="unexpected workflow_ref"): - github_certificate_identity( - workflow_ref="evil/repo/.github/workflows/publish.yml@refs/tags/v1.2.0" - ) + github_certificate_identity(workflow_ref="evil/repo/.github/workflows/publish.yml@refs/tags/v1.2.0") def test_production_tag_identity_matches_release_policy() -> None: assert production_tag_identity("v1.2.0") == ( - "https://github.com/fraware/open-verification-kernel/" - ".github/workflows/publish.yml@refs/tags/v1.2.0" + "https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.0" ) assert production_tag_identity("refs/tags/v1.2.0") == production_tag_identity("v1.2.0") diff --git a/tests/test_source_profile_hardening.py b/tests/test_source_profile_hardening.py new file mode 100644 index 0000000..b7b8699 --- /dev/null +++ b/tests/test_source_profile_hardening.py @@ -0,0 +1,140 @@ +"""Tests for Sprint 6 source-profile hardening beyond scaffolding.""" + +from __future__ import annotations + +from pathlib import Path + +from ovk.compilers.authorization import FastApiAstAuthorizationCompiler, materials_from_pair +from ovk.compilers.infrastructure import compile_kubernetes_objects, compile_terraform_plan +from ovk.core.source_profile_evidence import ( + collect_source_profile_evidence, + prove_actions_permissions_flow, + prove_fastapi_ast_profile, + prove_k8s_controller_profile, + prove_terraform_recursive_profile, +) +from ovk.core.source_profiles import compiler_binding_for, source_profile_strict_eligible +from ovk.core.template_conformance import EXECUTABLE_CATALOG, build_conformance_matrix + + +def test_fastapi_ast_detects_admin_bypass() -> None: + base = ( + "from fastapi import Depends, FastAPI\n" + "def require_admin():\n" + " return 'admin'\n" + "app = FastAPI()\n" + "@app.get('/admin/users', dependencies=[Depends(require_admin)])\n" + "def users():\n" + " return []\n" + ) + head = "from fastapi import FastAPI\napp = FastAPI()\n@app.get('/admin/users')\ndef users():\n return []\n" + materials = materials_from_pair(path="app.py", base_source=base, head_source=head) + ir = FastApiAstAuthorizationCompiler().compile(materials) + assert any("authorization.fastapi.ast_v1" in note for note in ir.warnings) + assert any(route.admin_only_before and not route.admin_only_after for route in ir.routes) + + +def test_fastapi_ast_marks_dynamic_path_unsupported() -> None: + source = ( + "from fastapi import FastAPI\napp = FastAPI()\npath = '/x'\n@app.get(path)\ndef handler():\n return {}\n" + ) + materials = materials_from_pair(path="app.py", base_source=source, head_source=source) + ir = FastApiAstAuthorizationCompiler().compile(materials) + assert any("dynamic_route_path" in item for item in ir.unsupported_constructs) + + +def test_terraform_recursive_child_modules() -> None: + plan = { + "format_version": "1.2", + "planned_values": { + "root_module": { + "resources": [], + "child_modules": [ + { + "address": "module.nested", + "resources": [ + { + "address": "module.nested.aws_s3_bucket.data", + "type": "aws_s3_bucket", + "name": "data", + "values": { + "tags": {"sensitivity": "confidential"}, + "acl": "public-read", + }, + } + ], + "child_modules": [], + } + ], + } + }, + } + ir = compile_terraform_plan(plan) + assert any("plan_recursive_v1" in note for note in ir.warnings) + assert any(resource.resource_id.endswith("aws_s3_bucket.data") for resource in ir.resources) + + +def test_kubernetes_controller_selector_edge() -> None: + objects = [ + { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": "api", "namespace": "default"}, + "spec": {"type": "LoadBalancer", "selector": {"app": "api"}}, + }, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": {"name": "api", "namespace": "default"}, + "spec": { + "template": { + "metadata": {"labels": {"app": "api"}}, + "spec": {"containers": [{"name": "api", "image": "api:1"}]}, + } + }, + }, + ] + ir = compile_kubernetes_objects(objects) + assert any(edge.kind == "service_selector" for edge in ir.edges) + assert any("controller_reachability_v1" in note for note in ir.warnings) + + +def test_profile_provers_and_bindings() -> None: + repo = Path(__file__).resolve().parents[1] + assert compiler_binding_for("authorization.fastapi.ast_v1") + fastapi = prove_fastapi_ast_profile(repo, enforcement_test="tests/test_authorization_enforcement.py") + assert fastapi.as_dict()["strict_eligible"] is True + tf = prove_terraform_recursive_profile(repo, enforcement_test="tests/test_remaining_lane_enforcement.py") + assert tf.as_dict()["strict_eligible"] is True + k8s = prove_k8s_controller_profile(repo, enforcement_test="tests/test_remaining_lane_enforcement.py") + assert k8s.as_dict()["strict_eligible"] is True + actions = prove_actions_permissions_flow(repo, enforcement_test="tests/test_remaining_lane_enforcement.py") + assert actions.as_dict()["strict_eligible"] is True + + +def test_template_v2_requires_semantic_evidence() -> None: + repo = Path(__file__).resolve().parents[1] + matrix = build_conformance_matrix(repo) + assert "source_profile_evidence" in matrix + by_id = {row["intent_id"]: row for row in matrix["templates"]} + auth = by_id["no-admin-route-bypass"] + assert auth["conformance_status_v2"] == "source_profile_strict_eligible" + assert auth["source_profile_evidence"]["strict_eligible"] is True + self_protection = by_id["agent-cannot-disable-own-ci-gate"] + assert self_protection["conformance_status_v2"] == "executable_advisory" + deployment = by_id["no-skipped-approval-state"] + # Deployment remains advisory until an explicit trusted_profile material exists. + assert deployment["conformance_status_v2"] == "executable_advisory" + assert matrix["counts_by_status_v2"].get("externally_calibrated_strict", 0) == 0 + + +def test_collect_evidence_covers_catalog_profiles() -> None: + repo = Path(__file__).resolve().parents[1] + evidence = collect_source_profile_evidence(repo, catalog_by_intent=EXECUTABLE_CATALOG) + assert "no-admin-route-bypass" in evidence + assert source_profile_strict_eligible( + profile_id=evidence["no-admin-route-bypass"].profile_id, + materials_trusted=evidence["no-admin-route-bypass"].materials_trusted, + coverage_complete=evidence["no-admin-route-bypass"].coverage_complete, + enforcement_test_present=evidence["no-admin-route-bypass"].enforcement_test_present, + ) diff --git a/tests/test_source_profiles.py b/tests/test_source_profiles.py new file mode 100644 index 0000000..06ec3c3 --- /dev/null +++ b/tests/test_source_profiles.py @@ -0,0 +1,38 @@ +"""Tests for Sprint 6 source-profile scaffolding.""" + +from __future__ import annotations + +from ovk.core.source_profiles import ( + is_known_source_profile, + profiles_from_policy, + source_profile_strict_eligible, +) + + +def test_known_profiles() -> None: + assert is_known_source_profile("authorization.fastapi.ast_v1") + assert not is_known_source_profile("made.up.profile") + + +def test_strict_eligible_requires_all_gates() -> None: + assert source_profile_strict_eligible( + profile_id="authorization.fastapi.ast_v1", + materials_trusted=True, + coverage_complete=True, + enforcement_test_present=True, + ) + assert not source_profile_strict_eligible( + profile_id="authorization.fastapi.ast_v1", + materials_trusted=False, + coverage_complete=True, + enforcement_test_present=True, + ) + + +def test_profiles_from_policy() -> None: + policy = { + "source_profiles": { + "authorization": ["authorization.fastapi.ast_v1", "bogus"], + } + } + assert profiles_from_policy(policy, lane="authorization") == ["authorization.fastapi.ast_v1"] diff --git a/tests/test_template_conformance.py b/tests/test_template_conformance.py index 5ca2039..7ee5d9b 100644 --- a/tests/test_template_conformance.py +++ b/tests/test_template_conformance.py @@ -93,3 +93,21 @@ def test_write_and_check_round_trip(tmp_path: Path) -> None: assert validate_matrix(matrix) == [] loaded = json.loads(output.read_text(encoding="utf-8")) assert loaded["template_count"] == matrix["template_count"] + + +def test_semantic_v2_statuses_from_profile_evidence() -> None: + repo = Path(__file__).resolve().parents[1] + matrix = build_conformance_matrix(repo) + assert set(matrix["conformance_statuses_v2"]) == { + "catalog_only", + "executable_advisory", + "source_profile_strict_eligible", + "externally_calibrated_strict", + "deprecated", + } + assert matrix["counts_by_status_v2"].get("externally_calibrated_strict", 0) == 0 + assert matrix["counts_by_status_v2"].get("source_profile_strict_eligible", 0) >= 1 + assert matrix["counts_by_status_v2"].get("executable_advisory", 0) >= 1 + by_id = {row["intent_id"]: row for row in matrix["templates"]} + assert by_id["no-admin-route-bypass"]["conformance_status_v2"] == "source_profile_strict_eligible" + assert by_id["agent-cannot-disable-own-ci-gate"]["conformance_status_v2"] == "executable_advisory" diff --git a/tests/test_trust_chain_r2_pr3_pr4_pr5.py b/tests/test_trust_chain_r2_pr3_pr4_pr5.py new file mode 100644 index 0000000..2a8455e --- /dev/null +++ b/tests/test_trust_chain_r2_pr3_pr4_pr5.py @@ -0,0 +1,320 @@ +"""Adversarial tests for OVK R2 trust-chain PR3–PR5.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from ovk.adapters.authorization import build_authorization_registry +from ovk.core.authorization_compiler import compile_authorization_obligation +from ovk.core.backend_aggregation import ( + FALLBACK_BLOCKING_TERMINATIONS, + aggregate_fail_dominant_v1, + evaluate_fallback_acceptance, +) +from ovk.core.backend_control_plane import BackendControlPlane +from ovk.core.execution_models import ( + BackendCapabilityAssessment, + BackendObligation, + BackendSelection, + ExecutionAttempt, + ExecutionBudget, + ExecutionContext, + FallbackPolicy, + NormalizedBackendResult, + RoutingDecision, +) +from ovk.core.models import MergeRecommendation, VerificationStatus +from ovk.core.router import RoutingConfig, route_obligation, select_primary_with_optional_corroboration +from ovk.core.self_protection_compiler import ( + compile_self_protection_obligation, + resolve_metadata_trusted, +) + + +def _budget() -> ExecutionBudget: + return ExecutionBudget( + total_wall_time_seconds=60, + per_backend_wall_time_seconds=30, + max_memory_mb=512, + max_parallel_backends=2, + allow_network=False, + allow_repository_write=False, + ) + + +def _assessment( + *, + backend: str = "test-backend", + support: str = "supported", + coverage_requirements_met: bool = True, + score: float = 0.9, +) -> BackendCapabilityAssessment: + return BackendCapabilityAssessment( + backend=backend, + support=support, # type: ignore[arg-type] + score=score, + guarantee_type="policy_evaluation", + material_requirements_met=True, + coverage_requirements_met=coverage_requirements_met, + native_available=False, + estimated_wall_time_seconds=5.0, + estimated_memory_mb=256, + reasons=["test assessment"], + ) + + +def test_incomplete_coverage_rejected_as_required_primary() -> None: + assessments = [_assessment(coverage_requirements_met=False)] + selected, rejected, eligible = select_primary_with_optional_corroboration( + assessments, + acceptable_guarantees=["policy_evaluation"], + config=RoutingConfig(accept_partial_primary=False), + budget=_budget(), + ) + assert selected == [] + assert any(item.reason == "coverage requirements not met" for item in rejected) + assert eligible == [] + + +def test_incomplete_coverage_eligible_optional_when_policy_allows() -> None: + assessments = [_assessment(coverage_requirements_met=False)] + selected, rejected, eligible = select_primary_with_optional_corroboration( + assessments, + acceptable_guarantees=["policy_evaluation"], + config=RoutingConfig(accept_partial_primary=True, max_selected_backends=2), + budget=_budget(), + ) + assert not any(item.required for item in selected) + assert rejected == [] + assert len(eligible) == 1 + assert eligible[0].support == "partial" + assert "incomplete coverage" in " ".join(eligible[0].reasons) + if selected: + assert all(not item.required for item in selected) + + +def test_guarantee_mismatch_fails_closed_without_rewrite() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_protected.json").read_text(encoding="utf-8")) + registry = build_authorization_registry() + obligation = compile_authorization_obligation(data, repo="r", head_sha="h") + budget = _budget() + routing = route_obligation( + obligation, + registry, + context=ExecutionContext(subject=obligation.subject, budget=budget, policy_digest="p"), + config=RoutingConfig(prefer_deterministic=True, max_selected_backends=1), + policy={"budget": {"allowed_backends": ["authorization-deterministic"]}}, + ) + routing = routing.model_copy( + update={ + "selected": [ + BackendSelection( + backend="authorization-deterministic", + reason="test", + expected_guarantee="smt_refutation_search", + required=True, + ) + ] + } + ) + + adapter = registry.require("authorization-deterministic") + original_compile = adapter.compile + + def _mismatching_compile( + compiled_obligation: Any, + compiled_routing: RoutingDecision, + ) -> BackendObligation: + compiled = original_compile(compiled_obligation, compiled_routing) + return compiled.model_copy(update={"expected_guarantee": "deterministic_witness"}) + + adapter.compile = _mismatching_compile # type: ignore[method-assign] + + record = BackendControlPlane(use_hardened_cache=False).execute( + obligation, + routing, + registry=registry, + cache=None, + ) + assert record.results[0].status == VerificationStatus.UNKNOWN + assert record.attempts[0].termination == "invalid_output" + assert record.merge_recommendation == MergeRecommendation.REQUIRE_HUMAN_REVIEW + assert "compiler guarantee mismatch" in record.results[0].limits[0] + + +@pytest.mark.parametrize("termination", sorted(FALLBACK_BLOCKING_TERMINATIONS)) +def test_blocking_terminations_never_accept_fallback(termination: str) -> None: + policy = FallbackPolicy( + allow_fallback=True, + fallback_backends=["authorization-deterministic"], + acceptable_fallback_guarantees=["deterministic_witness"], + ) + used, accepted, cause = evaluate_fallback_acceptance( + policy=policy, + selected=[ + BackendSelection( + backend="authorization-deterministic", + reason="test", + expected_guarantee="smt_refutation_search", + required=True, + ) + ], + attempts=[ + ExecutionAttempt( + attempt_id="att-1", + backend_obligation_id="bo-1", + backend="authorization-deterministic", + required=True, + started_at="2026-01-01T00:00:00Z", + finished_at="2026-01-01T00:00:01Z", + duration_ms=1.0, + termination=termination, # type: ignore[arg-type] + native_execution=False, + ) + ], + results=[ + NormalizedBackendResult( + attempt_id="att-1", + backend="authorization-deterministic", + status=VerificationStatus.PASS, + guarantee_type="deterministic_witness", + ) + ], + acceptable_guarantees=["smt_refutation_search"], + ) + assert used is True + assert accepted is False + assert cause == termination + + +def test_allow_fallback_without_blocking_cause_can_accept() -> None: + policy = FallbackPolicy( + allow_fallback=True, + fallback_backends=["authorization-deterministic"], + acceptable_fallback_guarantees=["deterministic_witness"], + ) + used, accepted, _ = evaluate_fallback_acceptance( + policy=policy, + selected=[ + BackendSelection( + backend="authorization-deterministic", + reason="test", + expected_guarantee="smt_refutation_search", + required=True, + ) + ], + attempts=[ + ExecutionAttempt( + attempt_id="att-1", + backend_obligation_id="bo-1", + backend="authorization-deterministic", + required=True, + started_at="2026-01-01T00:00:00Z", + finished_at="2026-01-01T00:00:01Z", + duration_ms=1.0, + termination="completed", + native_execution=False, + ) + ], + results=[ + NormalizedBackendResult( + attempt_id="att-1", + backend="authorization-deterministic", + status=VerificationStatus.PASS, + guarantee_type="deterministic_witness", + ) + ], + acceptable_guarantees=["smt_refutation_search"], + ) + assert used is True + assert accepted is True + + +def test_aggregate_fail_dominant_rejects_unaccepted_fallback_pass() -> None: + outcome = aggregate_fail_dominant_v1( + obligation_id="obl", + selected=[ + BackendSelection( + backend="a", + reason="test", + expected_guarantee="smt_refutation_search", + required=True, + ) + ], + results=[ + NormalizedBackendResult( + attempt_id="att-1", + backend="a", + status=VerificationStatus.PASS, + guarantee_type="deterministic_witness", + ) + ], + acceptable_guarantees=["smt_refutation_search"], + fallback_policy=FallbackPolicy(allow_fallback=False), + attempts=[ + ExecutionAttempt( + attempt_id="att-1", + backend_obligation_id="bo-1", + backend="a", + required=True, + started_at="2026-01-01T00:00:00Z", + finished_at="2026-01-01T00:00:01Z", + duration_ms=1.0, + termination="timeout", + native_execution=False, + ) + ], + ) + assert outcome.merge_recommendation == MergeRecommendation.REQUIRE_STRONGER_CHECK + assert outcome.fallback_used is True + assert outcome.fallback_accepted is False + assert outcome.fallback_cause == "timeout" + + +def test_metadata_trusted_defaults_false() -> None: + obligation = compile_self_protection_obligation( + { + "before": {"required_checks": ["ovk-verify"]}, + "after": {"required_checks": ["ovk-verify"]}, + }, + repo="r", + head_sha="h", + base_sha="b", + ) + assert obligation.abstraction["metadata_trusted"] is False + branch_materials = [item for item in obligation.materials if item.kind == "branch_protection"] + assert branch_materials + assert all(not item.trusted for item in branch_materials) + + +def test_resolve_metadata_trusted_requires_provenance_kind() -> None: + assert resolve_metadata_trusted(None) is False + assert resolve_metadata_trusted({}) is False + assert resolve_metadata_trusted({"trust": {"metadata_trusted": True}}) is False + assert ( + resolve_metadata_trusted( + { + "trust": { + "metadata_trusted": True, + "provenance_kind": "protected_base_workflow", + } + } + ) + is True + ) + + +def test_current_state_only_routing_flag_cannot_authorize_trust() -> None: + assert ( + resolve_metadata_trusted( + { + "routing": {"metadata_trusted": True}, + "trust": {"metadata_trusted": True}, + } + ) + is False + ) diff --git a/tests/test_typed_router.py b/tests/test_typed_router.py index d5daa32..17d6850 100644 --- a/tests/test_typed_router.py +++ b/tests/test_typed_router.py @@ -71,7 +71,12 @@ def _obligation(*, lane: str = "authorization") -> VerificationObligation: abstraction=abstraction, abstraction_digest=compute_abstraction_digest(abstraction), coverage=AbstractionCoverage(status="complete", confidence=1.0, extracted_elements=1), - acceptable_guarantees=["smt_refutation_search", "policy_evaluation", "workflow_secrets_boundary_check", "state_machine_safety"], + acceptable_guarantees=[ + "smt_refutation_search", + "policy_evaluation", + "workflow_secrets_boundary_check", + "state_machine_safety", + ], required_capabilities=[], policy_digest="policy-digest", ) diff --git a/tests/test_verification_cache.py b/tests/test_verification_cache.py index 7776fb3..ec73fba 100644 --- a/tests/test_verification_cache.py +++ b/tests/test_verification_cache.py @@ -12,6 +12,8 @@ BackendEnvironmentFingerprint, BackendObligation, BackendSelection, + CachedBackendExecution, + ExecutionAttempt, ExecutionBudget, FallbackPolicy, NormalizedBackendResult, @@ -24,6 +26,7 @@ from ovk.core.models import RiskSeverity, VerificationStatus, VerificationSubject from ovk.core.result_cache import ( CACHE_SCHEMA_VERSION, + CACHE_SCHEMA_VERSION_V2, NAMESPACE_BACKEND_RESULTS, HardenedResultCache, build_aggregate_key_components, @@ -43,7 +46,9 @@ def _budget() -> ExecutionBudget: ) -def _obligation(*, repo: str = "acme/api", head_sha: str = "abc", base_sha: str | None = "def") -> VerificationObligation: +def _obligation( + *, repo: str = "acme/api", head_sha: str = "abc", base_sha: str | None = "def" +) -> VerificationObligation: abstraction = {"kind": "test", "input": {"x": 1}} provisional = VerificationObligation( obligation_id="pending", @@ -118,6 +123,35 @@ def _fingerprint() -> BackendEnvironmentFingerprint: ) +def _cached_execution(result: NormalizedBackendResult, *, native: bool = False) -> CachedBackendExecution: + attempt = ExecutionAttempt( + attempt_id=result.attempt_id, + backend_obligation_id="bo-1", + backend=result.backend, + required=True, + started_at="2026-01-01T00:00:00Z", + finished_at="2026-01-01T00:00:01Z", + duration_ms=10.0, + termination="completed", + native_execution=native, + tool_version="0.1.0", + tool_digest="tool-digest", + exit_code=0, + raw_result_digest="raw-digest", + ) + return CachedBackendExecution( + attempt=attempt, + native_execution=native, + tool_version=attempt.tool_version, + tool_digest=attempt.tool_digest, + termination=attempt.termination, + exit_code=attempt.exit_code, + raw_result_digest=attempt.raw_result_digest, + environment_fingerprint="env-1", + normalized_result=result, + ) + + def test_key_components_include_required_fields() -> None: obligation = _obligation() routing = _routing(obligation) @@ -166,7 +200,7 @@ def test_subject_mismatch_is_cache_miss(tmp_path: Path) -> None: status=VerificationStatus.PASS, guarantee_type="exposure_graph_check", ) - cache.put_backend_result(components, result) + cache.put_backend_result(components, _cached_execution(result)) other = _obligation(repo="other/repo") other_components = build_backend_result_key_components( @@ -207,11 +241,13 @@ def test_routing_mismatch_is_cache_miss(tmp_path: Path) -> None: ) cache.put_backend_result( components, - NormalizedBackendResult( - attempt_id="a1", - backend="infrastructure-deterministic", - status=VerificationStatus.PASS, - guarantee_type="exposure_graph_check", + _cached_execution( + NormalizedBackendResult( + attempt_id="a1", + backend="infrastructure-deterministic", + status=VerificationStatus.PASS, + guarantee_type="exposure_graph_check", + ) ), ) mismatched = dict(components) @@ -298,11 +334,85 @@ def test_round_trip_backend_result(tmp_path: Path) -> None: guarantee_type="exposure_graph_check", counterexamples=[{"summary": "exposed"}], ) - cache.put_backend_result(components, result) + cache.put_backend_result(components, _cached_execution(result)) loaded = cache.get_backend_result(components) assert loaded is not None assert loaded.status == VerificationStatus.FAIL assert loaded.counterexamples == [{"summary": "exposed"}] + cached = cache.get_cached_execution(components) + assert cached is not None + assert cached.native_execution is False + assert cached.schema_version == CACHE_SCHEMA_VERSION + + +def test_v2_cache_entries_are_invalidated(tmp_path: Path) -> None: + cache = HardenedResultCache(tmp_path) + obligation = _obligation() + routing = _routing(obligation) + backend_obligation = _backend_obligation(obligation, routing) + components = build_backend_result_key_components( + obligation=obligation, + routing=routing, + backend_obligation=backend_obligation, + fingerprint=_fingerprint(), + ) + key = digest_key_components(components) + path = cache.namespace_dir(NAMESPACE_BACKEND_RESULTS) / f"{key}.json" + import json + + path.write_text( + json.dumps( + { + "cached_at": 1.0, + "key_digest": key, + "key_components": components, + "payload": { + "schema_version": CACHE_SCHEMA_VERSION_V2, + "normalized_result": { + "attempt_id": "a1", + "backend": "infrastructure-deterministic", + "status": "pass", + "guarantee_type": "exposure_graph_check", + "assumptions": [], + "limits": [], + "counterexamples": [], + "generated_artifacts": [], + }, + }, + "meta": {}, + } + ), + encoding="utf-8", + ) + assert cache.get_cached_execution(components) is None + assert not path.exists() + + +def test_cache_hit_preserves_native_provenance(tmp_path: Path) -> None: + """Adversarial: current tool availability must not rewrite cached native flag.""" + cache = HardenedResultCache(tmp_path) + obligation = _obligation() + routing = _routing(obligation) + backend_obligation = _backend_obligation(obligation, routing) + components = build_backend_result_key_components( + obligation=obligation, + routing=routing, + backend_obligation=backend_obligation, + fingerprint=_fingerprint(), + ) + result = NormalizedBackendResult( + attempt_id="native-a1", + backend="infrastructure-deterministic", + status=VerificationStatus.PASS, + guarantee_type="exposure_graph_check", + ) + cache.put_backend_result(components, _cached_execution(result, native=True)) + # Simulate environment where native is no longer available — hit must replay True. + loaded = cache.get_cached_execution(components) + assert loaded is not None + assert loaded.native_execution is True + assert loaded.attempt.native_execution is True + assert loaded.tool_digest == "tool-digest" def test_unknown_namespace_rejected(tmp_path: Path) -> None: diff --git a/tests/test_verified_source.py b/tests/test_verified_source.py index 1d2ea92..21e6d6e 100644 --- a/tests/test_verified_source.py +++ b/tests/test_verified_source.py @@ -1,23 +1,34 @@ -"""Tests for verified_source_sha resolution.""" +"""Tests for verified_source_sha / benchmark_source_sha resolution.""" from __future__ import annotations -from ovk.core.verified_source import resolve_verified_source_sha +from ovk.core.verified_source import resolve_benchmark_source_sha, resolve_verified_source_sha -def test_explicit_sha_wins(monkeypatch) -> None: +def test_explicit_verified_sha_wins(monkeypatch) -> None: monkeypatch.setenv("GITHUB_SHA", "github-sha") monkeypatch.setenv("OVK_VERIFIED_SOURCE_SHA", "env-sha") assert resolve_verified_source_sha(explicit="explicit-sha") == "explicit-sha" -def test_ovk_env_beats_github(monkeypatch) -> None: +def test_ovk_env_is_verified(monkeypatch) -> None: monkeypatch.setenv("GITHUB_SHA", "github-sha") monkeypatch.setenv("OVK_VERIFIED_SOURCE_SHA", "env-sha") assert resolve_verified_source_sha() == "env-sha" -def test_github_sha_used_when_no_override(monkeypatch) -> None: +def test_github_sha_is_not_verified(monkeypatch) -> None: monkeypatch.delenv("OVK_VERIFIED_SOURCE_SHA", raising=False) monkeypatch.setenv("GITHUB_SHA", "github-sha-only") - assert resolve_verified_source_sha() == "github-sha-only" + assert resolve_verified_source_sha() is None + + +def test_github_sha_is_benchmark_source(monkeypatch) -> None: + monkeypatch.delenv("OVK_BENCHMARK_SOURCE_SHA", raising=False) + monkeypatch.setenv("GITHUB_SHA", "github-sha-only") + assert resolve_benchmark_source_sha() == "github-sha-only" + + +def test_explicit_benchmark_sha_wins(monkeypatch) -> None: + monkeypatch.setenv("GITHUB_SHA", "github-sha") + assert resolve_benchmark_source_sha(explicit="bench-sha") == "bench-sha" diff --git a/tests/test_z3_obligations.py b/tests/test_z3_obligations.py index 5d9f24c..29e32ef 100644 --- a/tests/test_z3_obligations.py +++ b/tests/test_z3_obligations.py @@ -20,9 +20,7 @@ def test_admin_route_bypass_fixture_emits_counterexample() -> None: counterexamples = find_authorization_counterexamples(data) assert counterexamples assert counterexamples[0]["failure_mode"] == "admin_route_reachable_by_non_admin" - expected = json.loads( - Path("examples/z3_fail/admin_route_bypass.counterexample.json").read_text(encoding="utf-8") - ) + expected = json.loads(Path("examples/z3_fail/admin_route_bypass.counterexample.json").read_text(encoding="utf-8")) assert counterexamples[0]["route"] == expected["route"] assert counterexamples[0]["user_role"] == expected["user_role"] @@ -32,9 +30,7 @@ def test_privilege_escalation_fixture_emits_counterexample() -> None: counterexamples = find_privilege_escalation_counterexamples(data) assert counterexamples assert counterexamples[0]["failure_mode"] == "privilege_escalation" - expected = json.loads( - Path("examples/z3_fail/privilege_escalation.counterexample.json").read_text(encoding="utf-8") - ) + expected = json.loads(Path("examples/z3_fail/privilege_escalation.counterexample.json").read_text(encoding="utf-8")) assert counterexamples[0]["principal"] == expected["principal"] assert counterexamples[0]["gained_role"] == expected["gained_role"] diff --git a/tests/test_z3_validation.py b/tests/test_z3_validation.py index 58b072c..81d8f16 100644 --- a/tests/test_z3_validation.py +++ b/tests/test_z3_validation.py @@ -115,7 +115,4 @@ def test_validated_path_valid_input_does_not_emit_validation_failure() -> None: repo="example/repo", head_sha="abc", ) - assert all( - item.get("failure_mode") != "authorization_abstraction_invalid" - for item in evidence.counterexamples - ) + assert all(item.get("failure_mode") != "authorization_abstraction_invalid" for item in evidence.counterexamples)