From 35db991e0b8536a35c37b92ce070ab5868ee25c3 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:52:34 -0700 Subject: [PATCH 01/26] Document R2 deep audit and engineering program. Establish the authoritative post-control-plane audit and the ordered sprint/PR execution plan for the v1.3.0-rc.1 candidate without re-attributing signed v1.2.1. --- docs/DEEP_AUDIT_2026-07-23_R2.md | 189 ++++++++++++++++++++++ docs/ENGINEERING_PROGRAM_2026-07-23_R2.md | 181 +++++++++++++++++++++ 2 files changed, 370 insertions(+) create mode 100644 docs/DEEP_AUDIT_2026-07-23_R2.md create mode 100644 docs/ENGINEERING_PROGRAM_2026-07-23_R2.md diff --git a/docs/DEEP_AUDIT_2026-07-23_R2.md b/docs/DEEP_AUDIT_2026-07-23_R2.md new file mode 100644 index 0000000..d8a44f7 --- /dev/null +++ b/docs/DEEP_AUDIT_2026-07-23_R2.md @@ -0,0 +1,189 @@ +# OVK Deep Audit R2 — 2026-07-23 + +Authoritative deep audit of Open Verification Kernel after the typed backend control-plane landing. This document supersedes day-to-day use of [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) for release judgment. The vision audit remains historical context. + +Companion program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). Living dashboard: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). + +## Final technical judgment + +The engineers completed the most important architectural transition in the roadmap. + +OVK now has a genuine typed backend control plane. In explicitly enforced lanes, a typed `RoutingDecision` identifies selected registered adapters, and those selected adapters compile and execute backend-specific obligations. The system includes backend-neutral obligations, typed routing, controlled attempts, conservative aggregation, evidence v2, five enforceable lane paths, source-compiler profiles, template conformance, consumer repositories, and holdout plumbing. + +The vision is not fully achieved. Current code should be positioned as: + +**An advanced, policy-selectable verification-kernel release candidate for a future `v1.3.0-rc.1`.** + +The default product path remains shadow/legacy-authoritative. Enforced routing is available through lane policy, but several P0 trust properties remain incomplete. + +## Audit coverage + +This audit covered: + +- typed execution models and digest logic; +- backend registry, router, control plane, aggregation, and cache; +- all five enforced lanes; +- OPA, Z3, CBMC, Cedar, and external adapter surfaces; +- FastAPI, Express, Terraform, Kubernetes, GitHub Actions, deployment, and CBMC compilers; +- evidence v2, quality invariants, attestations, provenance, manifests, and Sigstore; +- GitHub Action, CI, release, holdout, and consumer workflows; +- FormalPR-Bench and template-conformance artifacts; +- both external consumer repositories. + +## Release provenance correction + +The signed `v1.2.1` release is commit: + +`a27d5720f4350c00bca34f71d991c31f5a2f38c7` + +Its release workflow successfully completed release verification, package build, isolated wheel smoke, and keyless Sigstore signing. + +That evidence applies only to the old tag. The typed control plane and the bulk of the current architecture were added after that commit. The two external consumer repositories also currently pin `v1.2.1`, so they validate the old release, not the newly pushed implementation. + +**Do not** re-attribute `v1.2.1` Sigstore/CI evidence to typed-control-plane commits. + +Latest `main` may be a benchmark badge commit marked `[skip ci]`. Generated benchmark data must not label a badge-only commit as `verified_source_sha` without a complete observed required-workflow set. + +Terminology required going forward: + +| Field | Meaning | +|---|---| +| `benchmark_source_sha` | Source measured by FormalPR-Bench (or similar bench artifacts) | +| `verified_source_sha` | Source with a complete observed required-workflow set | + +Badge-only or `[skip ci]` commits must never be labeled verified. + +## What has genuinely been achieved + +### Enforced backend execution + +Selected backends can now control execution within enforced lane paths. This is real, not merely metadata. + +Current selectable pairs include: + +- authorization through `z3-native` or `authorization-deterministic`; +- self-protection through `opa-native` or `self-protection-deterministic`; +- infrastructure, CI secrets, and deployment through registered deterministic adapters. + +### Strong execution and evidence foundations + +The repository now contains: + +- typed verification subjects and materials; +- backend-neutral obligations; +- backend capability assessments; +- typed routing decisions; +- backend-specific obligations; +- raw executions and normalized results; +- execution budgets; +- evidence v2; +- fail-dominant aggregation; +- routing- and environment-bound cache keys; +- release manifests, provenance, attestations, HMAC, and Sigstore support. + +### Honest catalog separation + +The template-conformance system distinguishes five linked production lanes from 95 catalog-only templates. This is substantially more honest than presenting all 100 templates as operational. Remaining gap: `strict_eligible` is still inferred mainly from repository links, fixtures, and a test file, rather than semantic completeness. + +### External consumer infrastructure exists + +The FastAPI/Terraform and Express/GitHub Actions consumer repositories are real and include pinned Action workflows, scenario matrices, pilot ledgers, release-bundle paths, and wheel-install scripts. Their automated scenarios remain separate from the required human-adjudicated pilot gate. + +## P0 defects remaining + +### 1. Attempt IDs are nondeterministic + +Execution attempt identity still includes `duration_ms`. Timing varies between otherwise equivalent runs, so attempt IDs, evidence, bundle IDs, and attestations can vary without a semantic input or output change. + +**Required fix:** remove timing fields from canonical attempt identity; prove stable IDs across sequential, parallel, cached, and uncached equivalent executions. + +### 2. Cache hits lose original execution provenance + +The cache returns normalized results without the complete original attempt. On a cache hit, the control plane synthesizes a new attempt and may re-infer `native_execution` from current tool availability. + +This can falsely describe a cached deterministic result as native after a tool is installed, or erase native tool provenance after the tool disappears. + +**Required fix:** store and replay `CachedBackendExecution` with original attempt, native flag, tool version/digest, termination/exit code, raw-result digest, environment fingerprint, and normalized result. Bump cache schema to v3. + +### 3. Guarantee mismatches are silently rewritten + +When the router expects one guarantee and the adapter compiler produces another, the control plane rewrites the compiled obligation to the router’s expectation. + +**Required fix:** treat mismatch as a compiler-contract error; skip execution; require review. Never silent `model_copy` rewrite. + +### 4. Coverage does not govern backend eligibility + +Adapters report `coverage_requirements_met`, but required-primary selection does not reject candidates for which this field is false. + +**Required fix:** reject incomplete candidates as required primaries; allow only as optional corroborators under explicit policy. Stop hardcoding `coverage_requirements_met=True` in manifest assessment. + +### 5. Fallback policy is under-specified + +Aggregation receives a broad `fallback_accepted` boolean. It does not constrain fallback by backend, guarantee type, or failure cause. + +**Required fix:** backend-, guarantee-, and cause-specific policy. Native timeout, tool error, invalid output, and resource exhaustion must never become passing fallback results. + +### 6. Self-protection metadata defaults to trusted + +Enforced self-protection currently initializes `metadata_trusted` to true unless policy explicitly sets it false. + +**Required fix:** default `metadata_trusted=False`. Trust only from protected base-workflow provenance, signed service, or explicit maintainer-supplied material. Current-state-only branch protection cannot set trusted. + +### 7. Deterministic adapters cannot be hard-cancelled + +Several deterministic adapters execute in-process and inspect elapsed time only after returning. A hung or unexpectedly expensive evaluator cannot be terminated. + +**Required fix:** every authoritative adapter must run behind a subprocess/spawned-worker boundary with wall-time, output, env, path, and hard cancellation. + +### 8. Two routing paths remain + +The kernel calculates compatibility routing before compiling obligations. The enforced runtime later compiles a typed obligation and calculates another typed route. + +**Required fix:** compile typed obligations before routing; route each obligation exactly once; execute that immutable `RoutingDecision`; expose the same `routing_id` through kernel, CLI, MCP, evidence, provenance, and attestation. + +### 9. Evidence v2 does not require the complete trace + +The schema requires obligation ID, routing ID, selected and executed backends, and aggregation policy. It leaves compiler, materials, coverage, requested/eligible/attempted backends, execution attempts, and routing-enforced state optional or broadly typed. + +**Required fix:** evidence v3 with mandatory full control-plane trace and one canonical material-set digest cross-bound across obligation, evidence, provenance, attestation, and release verification. + +## Direct fixes required with this audit baseline + +Claimed during audit authoring; must be present in tree before Sprint 0 measurement: + +1. **CI-secrets / material integrity** — `size_bytes` must bind canonical serialized payload length, not digest string length. +2. **Backend worker environment** — inherit only configured safe parent vars; strip credentials; reject non-positive wall budget. +3. **FormalPR-Holdout supply-chain boundary** — immutable SHA-256, path-safe extraction, isolated Python, no tokens in evaluator, schema validation, leakage guards. + +## Definition of completed vision (18-condition gate) + +Ship / promote to `v1.3.0` only when all hold: + +1. one route per enforced obligation; +2. the same `routing_id` across kernel, evidence, provenance, and attestation; +3. coverage-aware selection; +4. no silent guarantee mismatch; +5. constrained fallback; +6. provenance-preserving cache hits; +7. deterministic attempt identities; +8. hard-bounded adapters; +9. protected self-protection trust; +10. complete evidence trace; +11. cross-artifact material binding; +12. semantic template conformance; +13. current-source CI on a non-`[skip ci]` SHA; +14. current wheel and Action validation in both consumers; +15. label-separated holdout predictions; +16. immutable retained evaluation artifacts; +17. attributable release gates on the exact tag source; +18. correct `benchmark_source_sha` vs `verified_source_sha` terminology in published artifacts. + +## Related documents + +| Document | Role | +|---|---| +| [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md) | Sprint/PR execution program | +| [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) | Living adoption dashboard | +| [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) | Historical pre-control-plane audit | +| [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) | Consumer pin checklist | +| [FORMALPR_HOLDOUT_GOVERNANCE.md](FORMALPR_HOLDOUT_GOVERNANCE.md) | Holdout governance | diff --git a/docs/ENGINEERING_PROGRAM_2026-07-23_R2.md b/docs/ENGINEERING_PROGRAM_2026-07-23_R2.md new file mode 100644 index 0000000..5fbf887 --- /dev/null +++ b/docs/ENGINEERING_PROGRAM_2026-07-23_R2.md @@ -0,0 +1,181 @@ +# OVK Engineering Program R2 — 2026-07-23 + +Execution program derived from [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Release positioning: future **`v1.3.0-rc.1`** candidate. Do not re-validate signed `v1.2.1` (`a27d572…`) as if it included the typed control plane. + +## Mandatory constraints + +- Do not merge trust-chain PRs out of order (identity → cache → contracts → single route → isolation → evidence). +- Every PR must include the eight analysis sections below and fail-closed adversarial tests. +- Sprints 6–7 may overlap late PRs only after PR6 lands. +- External consumer and holdout work needing write access outside this repo is coordinated separately; this repo carries pins, checklists, and scaffolding. + +## Per-PR analysis checklist (required) + +Every PR in this program must document: + +1. **Architecture** — what trust boundary or control-plane invariant changes. +2. **Schema** — which schemas/versions bump and why. +3. **Cache** — key/value identity impact; invalidation or migration. +4. **Migration** — how old artifacts/entries fail closed or upgrade. +5. **Trust-boundary** — what an adversary can no longer forge or confuse. +6. **Adversarial tests** — explicit negative tests that fail closed. +7. **Artifacts** — which evidence/provenance/attestation/release fields change. +8. **Docs** — status, roadmap, and operator-facing honesty updates. + +## Sprint 0 — Current-source baseline + +On one **non-`[skip ci]`** source SHA, run and retain: + +- general CI, native Tier 1, package/wheel smoke outside checkout; +- Action dogfood, release preflight, expanded FormalPR-Bench; +- template conformance, adversarial release-bundle checks. + +Record exact workflow run URLs in [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). + +Introduce `benchmark_source_sha` alongside `verified_source_sha` in badge/summary renderers and schemas. `verified_source_sha` requires a complete observed required-workflow set; badge-only commits must not be labeled verified. + +## Sprint 1 — Attempt identity and cache provenance + +### PR1 — Deterministic attempt identity + +**Targets:** `ovk/core/execution_models.py` (`attempt_digest_input` / `compute_attempt_id`) + +**Acceptance:** + +- Remove `duration_ms` (and any other timing) from canonical attempt identity; keep duration as observational metadata only. +- Prove stable IDs across sequential, parallel, cached, and uncached equivalent runs. +- Remove/replace vacuous tests that assert timing participates in identity. + +### PR2 — Provenance-preserving cache v3 + +**Targets:** `ovk/core/result_cache.py`, `ovk/core/backend_control_plane.py` cache-hit branch + +**Acceptance:** + +- Introduce `CachedBackendExecution` storing original `ExecutionAttempt`, `native_execution`, tool version/digest, termination/exit code, raw-result digest, environment fingerprint, normalized result. +- On hit: **replay** stored attempt (do not synthesize; do not re-infer `native_execution` from current tool availability). +- Bump schema to `ovk.cache.v3`; migrate or invalidate v2 entries explicitly. +- Adversarial tests: install/remove native tool between write and hit; prove provenance unchanged. + +## Sprint 2 — Routing-contract enforcement + +### PR3 — Coverage and guarantee enforcement + +**Targets:** `ovk/core/router.py`, `ovk/core/backend_control_plane.py` + +**Acceptance:** + +- Reject candidates with `coverage_requirements_met=False` as required primaries; allow only as optional corroborators under explicit policy. +- Stop hardcoding `coverage_requirements_met=True` in `_manifest_assessment`. +- On compiler vs router `expected_guarantee` mismatch: compiler-contract error, skip execution, require review — never silent rewrite. + +### PR4 — Fallback policy v2 + +**Targets:** `ovk/core/execution_models.py` (`FallbackPolicy`), `ovk/core/backend_aggregation.py`, evidence decision fields for INV-017 + +**Acceptance:** + +- Replace broad `fallback_accepted=allow_fallback` with backend-, guarantee-, and cause-specific policy. +- Native timeout, tool error, invalid output, and resource exhaustion must never become passing fallback results. + +### PR5 — Self-protection trust provenance + +**Targets:** `ovk/core/adapter_runtime.py`, `ovk/core/self_protection_compiler.py` + +**Acceptance:** + +- Default `metadata_trusted=False`. +- Trust only from protected base-workflow provenance, signed service, or explicit maintainer-supplied material. +- Current-state-only branch protection cannot set trusted. + +## Sprint 3 — One authoritative route + +### PR6 — Single authoritative routing pipeline + +**Targets:** `ovk/core/router.py`, kernel/CLI/MCP planning surfaces, evidence/provenance/attestation emitters + +**Acceptance:** + +- Compile typed obligations **before** routing; route each obligation exactly once; execute that immutable `RoutingDecision`. +- Eliminate dual outcomes from `route_intent` (compat) vs `route_obligation` (typed). +- Same `routing_id` across kernel, CLI, MCP, evidence, provenance, attestation. + +## Sprint 4 — Hard adapter isolation + +### PR7 — Isolated deterministic workers + +**Targets:** five `*DeterministicAdapter.run` classes; `ovk/core/execution_budget.py` + +**Acceptance:** + +- Move deterministic evaluators behind subprocess/spawned-worker boundary with wall-time, output, env, path, and hard cancellation. +- Control plane always passes worker; in-process-only authoritative adapters are forbidden. + +### PR8 — Remaining adapter isolation + +**Acceptance:** + +- Native OPA/Z3/CBMC/Cedar paths share the same externally enforced worker contract. +- No authoritative adapter may inspect elapsed time only after returning. + +## Sprint 5 — Evidence v3 and material binding + +### PR9 — Evidence v3 and material-set binding + +**Targets:** evidence schema v3, `ovk/core/evidence_invariants.py`, provenance/attestation/release verification + +**Acceptance:** + +- Require full control-plane trace: compiler, materials, coverage, requested/eligible/attempted backends, execution attempts, routing-enforced state. +- Cross-bind one canonical **material-set digest** across obligation, evidence, provenance, attestation, and release verification; recompute identities during validation. + +## Sprint 6 — Source-profile hardening + +AST/module-graph authorization profiles; recursive Terraform plans; controller-aware K8s reachability; deeper Actions permissions/secret flow; deployment strictness only on explicit trusted profiles. Touch lane compilers under `ovk/core/*_compiler.py` and source-profile modules. + +Parallelizable after PR6. + +## Sprint 7 / PR10 — Semantic template conformance v2 + +Replace file-existence `strict_eligible` generation with: + +`catalog_only` | `executable_advisory` | `source_profile_strict_eligible` | `externally_calibrated_strict` | `deprecated` + +Every status must derive from executed semantic evidence, not repo-link/fixture/test-file presence. + +## Sprint 8 — Label-separated holdout + +Generate predictions from the exact RC artifact **without** labels; sign/digest; evaluate separately with protected labels; publish aggregates only. Builds on Phase A holdout hardening. + +## Sprint 9 — Consumer validation on current code + +Update both consumers from `v1.2.1` to immutable `v1.3.0-rc.1` (or audited commit): + +- https://github.com/fraware/ovk-consumer-fastapi-terraform +- https://github.com/fraware/ovk-consumer-express-actions + +Dispatch workflows, download evidence, verify bundles, exercise true cross-fork PRs; keep human pilot ledgers separate from automated fixtures. Update [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) pins in this repo; consumer repo PRs require write access. + +## Sprint 10 — Attributable publication + +- Correct benchmark vs verified-source terminology in published artifacts. +- Require all automated release gates on the exact tag source. +- Sign artifacts; retain workflow IDs and digests. +- Promote to `v1.3.0` only after P0 closure (PRs 1–9), consumer validation, and attributable holdout aggregates. + +## Required PR order (summary) + +1. Deterministic attempt identity +2. Provenance-preserving cache v3 +3. Coverage and guarantee enforcement +4. Fallback policy v2 +5. Self-protection trust provenance +6. Single authoritative routing pipeline +7. Isolated deterministic workers +8. Remaining adapter isolation +9. Evidence v3 and material-set binding +10. Template conformance v2 + +## Phase A prerequisite (before Sprint 0 measurement) + +Author this document and the deep audit; rewrite [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) for `v1.3.0-rc.1`; land CI-secrets `size_bytes`, worker env allowlist + zero-budget reject, and FormalPR-Holdout isolation hotfixes. From d54ddc51d043eeba4cc35b551bdf0ae4f3b66f33 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:52:34 -0700 Subject: [PATCH 02/26] Reposition docs for v1.3.0-rc.1 candidate status. Point the adoption dashboard, roadmap, and doc index at the R2 audit/program so operators do not treat typed control-plane work as covered by signed v1.2.1. --- docs/CURRENT_RELEASE_STATUS.md | 203 ++++++++++++++++++++------------ docs/EXPERIMENTAL_PATHS.md | 8 +- docs/README.md | 9 +- docs/RELEASE.md | 2 +- docs/ROADMAP.md | 13 +- docs/VISION_AUDIT_2026-07-22.md | 2 + 6 files changed, 149 insertions(+), 88 deletions(-) diff --git a/docs/CURRENT_RELEASE_STATUS.md b/docs/CURRENT_RELEASE_STATUS.md index 63eb7d7..71efc75 100644 --- a/docs/CURRENT_RELEASE_STATUS.md +++ b/docs/CURRENT_RELEASE_STATUS.md @@ -1,77 +1,126 @@ -# OVK Release Status - -Living adoption dashboard for Open Verification Kernel. - -**Last updated:** 2026-07-23 - -**Release judgment:** v1.2.1 **release candidate**. The bounded evidence pipeline is substantial, but the complete solver-agnostic kernel vision is still partial because backend routing enforcement is landing in the control-plane stack and still requires independent consumer validation. Independent tagged-consumer validation remains required before calling the release production-stable. - -Authoritative audit: [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md). - -## At a glance - -| Signal | Current state | -|---|---| -| **Package version** | `1.2.1` release candidate; immutable tag `v1.2.1` cut; PyPI `publish` job was skipped on the release run and must be confirmed separately | -| **FormalPR-Bench** | Repository snapshot reports 130/130 curated regression cases; this is internal conformance, not an external accuracy estimate | -| **Check types** | Five bounded production lanes: self-protection, authorization, infrastructure, CI secrets, deployment | -| **Backend execution** | OPA and Z3 native paths; CBMC bounded explicit/template harness path; Cedar version probe plus deterministic evaluator; six deterministic contract adapters | -| **Routing** | Control-plane routing can enforce backend selection for production lanes; see template conformance and lane enforcement tests | -| **Unit and workflow tests** | Release Publish `verify` green on tag `v1.2.1` ([run 30010876652](https://github.com/fraware/open-verification-kernel/actions/runs/30010876652)); CI uses `unit` then `gates` jobs — cite `verified_source_sha` for badge `[skip ci]` commits | -| **Package portability** | Wheel-outside-checkout smoke passed in Publish `verify` for `v1.2.1` | -| **GitHub Action** | Automatic PR-diff collection and quoted arguments are implemented; independent tagged consumer repository remains pending | -| **External validation** | Current workflow is in-repository dogfooding; external pilot registry contains no completed independent pilot evidence yet | -| **Sigstore** | Immutable-tag keyless E2E closed for `v1.2.1` (see gap #9) | -OVK is not complete formal verification of arbitrary code. It provides explainable, conservative checks for a bounded set of high-risk changes and emits explicit unknown and human-review outcomes. - -## Adoption readiness - -| Mode | Current recommendation | Conditions | -|---|---|---| -| **Local/demo** | Appropriate after current CI is green | Use shipped examples and inspect assumptions and limits | -| **Advisory Action** | Appropriate for pilots after current CI is green | Collect adjudicated false positives, unknowns, and missed detections | -| **Strict required check** | Repository-specific only | Calibrate on real diffs; use trusted abstraction sources and protected policy metadata | -| **Production-stable general enforcement** | Not yet | Requires enforced backend routing, source-grounded compilers, independent pilots, and attributable release CI | - -Suggested rollout: local validation → advisory artifacts → advisory check run/comment → calibrated strict lane → protected required check. - -## Current high-priority gaps - -1. Backend routing is advisory metadata and does not control compilation or execution. -2. The 100-template catalog does not equal 100 executable, end-to-end properties. -3. Authorization, infrastructure, CI workflow, and deployment diff extraction remain heuristic. -4. Cedar and six other external adapters do not perform native proof/policy execution. -5. FormalPR-Bench is an internal curated regression corpus without an independent holdout. -6. No completed independent repository currently proves the tagged Action and wheel integration. -7. Current-commit CI evidence is not attached to the latest observed `[skip ci]` HEAD. -8. Auto-collected branch protection cannot reconstruct removed required checks without trusted before/after data. -9. Live Sigstore keyless signing: protected `workflow_dispatch` dry-run succeeded ([run 30008891551](https://github.com/fraware/open-verification-kernel/actions/runs/30008891551)). **Immutable-tag Release E2E is closed** for `v1.2.1` ([release](https://github.com/fraware/open-verification-kernel/releases/tag/v1.2.1); [Publish run 30010876652](https://github.com/fraware/open-verification-kernel/actions/runs/30010876652); identity `@refs/tags/v1.2.1`, release-attached `*.cosign.bundle.json`). PyPI `publish` was skipped on that run and remains a separate follow-up — see [RELEASE.md](RELEASE.md#sigstorecosign-keyless). - -Full analysis and acceptance criteria: [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md). - -## Maintainer release gates - -Before tagging or publishing v1.2.0: - -- [ ] run all CI and native Tier 1 jobs on a non-`[skip ci]` source commit; -- [ ] confirm wheel smoke from a directory outside the checkout; -- [ ] confirm automatic-diff composite Action dogfood; -- [ ] confirm package version matches the release tag; -- [ ] run full expanded FormalPR-Bench and release preflight; -- [ ] validate a complete release bundle, including evidence-quality semantics; -- [ ] exercise HMAC signing and identity-bound Sigstore signing according to release policy; -- [ ] run the immutable Action or release wheel in an independent consumer repository; -- [ ] update status with exact source SHA and workflow links; -- [ ] keep the package classifier at Beta until independent pilots and backend-routing enforcement meet the production gate. - -## Related documents - -| Document | Purpose | -|---|---| -| [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) | Current deep code, artifact, vision, and engineering audit | -| [STATUS.md](STATUS.md) | Command and lane inventory | -| [BACKENDS.md](BACKENDS.md) | Exact backend execution maturity and guarantee classes | -| [INTEGRATION.md](INTEGRATION.md) | Installation and GitHub Action setup | -| [RELEASE.md](RELEASE.md) | Maintainer release procedure | -| [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) | Independent advisory pilot process | -| [BENCHMARK.md](BENCHMARK.md) | Internal benchmark format and execution | +# OVK Release Status + +Living adoption dashboard for Open Verification Kernel. + +**Last updated:** 2026-07-23 + +**Release judgment:** **`v1.3.0-rc.1` candidate**. The typed backend control plane post-dates signed `v1.2.1` (`a27d5720f4350c00bca34f71d991c31f5a2f38c7`). Default product path remains shadow/legacy-authoritative; enforced routing is lane-policy opt-in until P0 trust closure. Do not treat current `main` as a re-validation of signed `v1.2.1`. + +Authoritative audit: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Engineering program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). Historical: [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) (superseded for day-to-day status). + +## At a glance + +| Signal | Current state | +|---|---| +| **Package version** | Working tree targets future `v1.3.0-rc.1`; signed immutable tag remains `v1.2.1` only for that tag’s commit | +| **FormalPR-Bench** | Internal curated regression; report `benchmark_source_sha` separately from `verified_source_sha` | +| **Check types** | Five bounded production lanes: self-protection, authorization, infrastructure, CI secrets, deployment | +| **Backend execution** | Typed `BackendControlPlane` + `route_obligation`; five policy-selectable enforced lanes via `adapter_runtime` | +| **Routing** | Enforced under lane policy; default path still shadow/legacy-authoritative until P0 closure | +| **Unit and workflow tests** | Local Sprint 0 baseline recorded below; live GitHub Actions workflow IDs still pending | +| **Package portability** | Local wheel-outside-checkout import smoke passed on working tree (package metadata still `1.2.1` until rc.1 cut) | +| **GitHub Action** | Consumers still live-pin `v1.2.1`; local consumer clones prepared for `v1.3.0-rc.1` (not pushed) | +| **External validation** | In-repository dogfooding + consumer scaffolding; independent pilots incomplete | +| **Sigstore** | Immutable-tag E2E closed for `v1.2.1` only — not attributable to typed control-plane commits | + +OVK is not complete formal verification of arbitrary code. It provides explainable, conservative checks for a bounded set of high-risk changes and emits explicit unknown and human-review outcomes. + +## Source SHA terminology + +| Field | Meaning | When to set | +|---|---|---| +| `benchmark_source_sha` | Commit whose FormalPR-Bench (or badge) artifacts were measured | Any bench/badge run | +| `verified_source_sha` | Commit with a **complete observed required-workflow set** | Only after Sprint 0 / release gates attach live workflow IDs | + +Badge-only or `[skip ci]` commits must set `benchmark_source_sha` and must **not** be labeled `verified_source_sha`. + +## Local Sprint 0 baseline + +Local evidence only. Distinguishes from GitHub Actions workflow IDs (still pending). Working tree HEAD at measurement time: `4b48ab245193e177a6d95e8557332334a9bd2883` (badge `[skip ci]` tip — treat as `benchmark_source_sha`, not verified). + +| Gate | Command | Exit | Timestamp (local) | +|---|---|---|---| +| Focused R2 + enforcement pytest | `python -m pytest tests/test_source_profile_hardening.py tests/test_template_conformance.py tests/test_verified_source.py tests/test_bench_badge.py tests/test_formalpr_holdout_runner.py tests/test_execution_models.py tests/test_cache_worker_control_plane.py tests/test_adapter_isolation_r2_pr8.py tests/test_evidence_v3_r2_pr9.py tests/test_authorization_enforcement.py tests/test_adversarial_control_plane.py tests/test_source_profiles.py -q` | **0** (111 passed) | 2026-07-23T23:41:09-07:00 → 23:42:03 | +| Broader compiler/cache suite | `python -m pytest tests/test_authorization_compilers.py tests/test_infrastructure_compilers.py tests/test_github_actions_trust.py tests/test_remaining_lane_enforcement.py tests/test_self_protection_enforcement.py tests/test_verification_cache.py tests/test_result_cache_semantics.py -q` | **0** (54 passed) | 2026-07-23T23:35:44-07:00 | +| Sprint 6–8 regression | `python -m pytest tests/test_source_profile_hardening.py tests/test_template_conformance.py tests/test_formalpr_holdout_runner.py tests/test_source_profiles.py tests/test_authorization_compilers.py tests/test_infrastructure_compilers.py -q` | **0** (46 passed) | 2026-07-23T23:40:48-07:00 | +| Release preflight (`PYTHONPATH=.`) | `python scripts/release_preflight.py` | **0** | 2026-07-23T23:44:17-07:00 → 23:45:10 | +| Template validation | `python scripts/validate_templates.py` | **0** | 2026-07-23T23:41:09-07:00 | +| Template conformance v2 regenerate | `python scripts/build_template_conformance.py` | **0** (`source_profile_strict_eligible=3`, `executable_advisory=2`, `catalog_only=95`) | 2026-07-23T23:40:57-07:00 | +| Local release smoke | `python scripts/smoke_release_local.py` | **0** | 2026-07-23T23:41:09-07:00 | +| Wheel build + outside-checkout import | `python -m build --wheel` then `pip install … -t $TEMP/ovk-outside-import` and `import ovk` | **0** (`verified_source_sha` correctly `None` outside attested env) | 2026-07-23T23:46:01-07:00 → 23:46:29 | +| Workflow ID collector | `python scripts/collect_workflow_evidence.py --sha --output .verification/workflow-evidence-local.json` | **0** (0 runs on `[skip ci]` tip; `verified_source_sha` left unset) | 2026-07-23T23:46:29-07:00 | + +### Still pending (live GitHub Actions / secrets) + +| Gate | Status | Evidence | +|---|---|---| +| General CI / unit+gates on non-`[skip ci]` SHA | Pending live run | Record run URL when available | +| Native Tier 1 | Pending | — | +| Action dogfood | Pending | — | +| Expanded FormalPR-Bench on release SHA | Pending | Use `benchmark_source_sha` | +| Adversarial release-bundle in Actions | Pending | Local `verify_release_bundle.py` entrypoint present | +| Label-separated holdout live eval | Pending | Needs `HOLDOUT_DOWNLOAD_TOKEN` + `HOLDOUT_ASSET_SHA256` | +| Consumer remotes on rc.1 | Pending push | Local clones prepared under `%TEMP%\ovk-consumer-prep\` (not pushed) | + +## Adoption readiness + +| Mode | Current recommendation | Conditions | +|---|---|---| +| **Local/demo** | Appropriate after current local/CI green | Use shipped examples and inspect assumptions and limits | +| **Advisory Action** | Appropriate for pilots on pinned tags | Prefer `v1.2.1` until rc.1 is attributable; collect FPs/unknowns | +| **Strict required check** | Repository-specific only | Calibrate on real diffs; trusted abstraction sources; protected policy metadata | +| **Production-stable general enforcement** | Not yet | P0 code (PRs 1–9) in working tree; still needs consumers on rc.1, attributable holdout, and Sprint 0 live gates | + +Suggested rollout: local validation → advisory artifacts → advisory check run/comment → calibrated strict lane → protected required check. + +## P0 trust defects (R2 PRs 1–9) — working-tree status + +Code for PRs 1–9 is present in this working tree (attempt identity excludes `duration_ms`; `ovk.cache.v3` / `CachedBackendExecution`; coverage/guarantee fail-closed; fallback v2 blocking terminations; `metadata_trusted` default false; authoritative routing pipeline; worker isolation; `ovk.evidence.v3` material-set binding). Historical defect inventory: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). + +**Still open for attributable release (external / Sprint 0–10):** live non-`[skip ci]` workflow IDs, consumer repo pins on immutable rc.1, label-separated holdout aggregates, and signed publication gates — see checklists below and [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). + +## Sprint 6–10 working-tree progress + +| Sprint | Local status | +|---|---| +| 6 Source profiles | FastAPI AST compiler; Terraform recursive modules; K8s controller reachability; Actions permissions-flow prover; deployment trusted-profile gate | +| 7 Template conformance v2 | Statuses derived from executed profile evidence (`source_profile_strict_eligible=3`; no `externally_calibrated_strict` from local gen) | +| 8 Holdout separation | `digest_holdout_predictions.py` + label-free guards; eval path token-stripped | +| 9 Consumers | In-repo template + checklist for rc.1; local clones pin-prepped (no push) | +| 10 Publication | [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md) + `scripts/collect_workflow_evidence.py` | + +## Maintainer release gates + +Before tagging or publishing **`v1.3.0-rc.1`**: + +- [ ] run all CI and native Tier 1 jobs on a non-`[skip ci]` source commit; +- [ ] confirm wheel smoke from a directory outside the checkout; +- [ ] confirm automatic-diff composite Action dogfood; +- [ ] confirm package version matches the release tag; +- [ ] run full expanded FormalPR-Bench and release preflight; +- [ ] validate a complete release bundle, including evidence-quality semantics; +- [ ] exercise HMAC signing and identity-bound Sigstore signing according to release policy; +- [ ] run the immutable Action or release wheel in both independent consumer repositories at the rc.1 pin; +- [ ] update status with exact `verified_source_sha` and workflow links; +- [ ] confirm P0 trust PRs 1–9 on the exact tag source and record attributable holdout aggregates; +- [ ] keep the package classifier at Beta until independent pilots and P0 closure meet the production gate. + +Promotion to **`v1.3.0`** additionally requires P0 closure + consumer + holdout evidence per [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). Do not re-attribute `v1.2.1` Sigstore evidence to typed-control-plane commits. + +## Related documents + +| Document | Purpose | +|---|---| +| [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | Authoritative R2 deep audit | +| [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md) | Sprint/PR execution program | +| [SOURCE_PROFILE_HARDENING.md](SOURCE_PROFILE_HARDENING.md) | Sprint 6 profile status | +| [HOLDOUT_LABEL_SEPARATION.md](HOLDOUT_LABEL_SEPARATION.md) | Sprint 8 prediction/eval split | +| [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) | Sprint 9 consumer pins | +| [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md) | Sprint 10 publication gate | +| [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) | Historical pre-control-plane audit | +| [STATUS.md](STATUS.md) | Command and lane inventory | +| [BACKENDS.md](BACKENDS.md) | Exact backend execution maturity and guarantee classes | +| [INTEGRATION.md](INTEGRATION.md) | Installation and GitHub Action setup | +| [RELEASE.md](RELEASE.md) | Maintainer release procedure | +| [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) | Independent advisory pilot process | +| [BENCHMARK.md](BENCHMARK.md) | Internal benchmark format and execution | diff --git a/docs/EXPERIMENTAL_PATHS.md b/docs/EXPERIMENTAL_PATHS.md index fb284a8..61ece26 100644 --- a/docs/EXPERIMENTAL_PATHS.md +++ b/docs/EXPERIMENTAL_PATHS.md @@ -56,6 +56,8 @@ mismatches are cache misses — stale results must not authorize allow. ## Metric provenance -Generated badge/summary/adoption metrics must carry `verified_source_sha` for -the commit that produced the numbers. Later `[skip ci]` badge commits must not -be cited as the verified source. See program §24.3. +Generated badge/summary/adoption metrics must carry `benchmark_source_sha` for +the commit that produced the numbers. Set `verified_source_sha` only when a +complete required-workflow set was observed. Later `[skip ci]` badge commits must +not be cited as the verified source. See program §24.3 and +[CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). diff --git a/docs/README.md b/docs/README.md index 805c982..5b95b54 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # OVK Documentation -Documentation for Open Verification Kernel **v1.2.0**. +Documentation for Open Verification Kernel (**v1.3.0-rc.1 candidate**; signed tag remains `v1.2.1`). Use this index as the canonical entry point. Each guide covers one topic; cross-links replace duplicated content across files. @@ -11,13 +11,15 @@ Use this index as the canonical entry point. Each guide covers one topic; cross- | **Adopting in CI** | [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) → [INTEGRATION.md](INTEGRATION.md) → [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) | | **Contributing code** | [CONTRIBUTING.md](CONTRIBUTING.md) → [ARCHITECTURE.md](ARCHITECTURE.md) → [ADAPTER_CONTRACT.md](ADAPTER_CONTRACT.md) | | **Maintainers** | [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) → [RELEASE.md](RELEASE.md) → `ovk release-preflight` (release readiness checks) | -| **Spec / security review** | [SYSTEM_SPEC.md](SYSTEM_SPEC.md) → [FORMAL_SPEC.md](FORMAL_SPEC.md) → [THREAT_MODEL.md](THREAT_MODEL.md) | +| **Spec / security review** | [SYSTEM_SPEC.md](SYSTEM_SPEC.md) → [FORMAL_SPEC.md](FORMAL_SPEC.md) → [THREAT_MODEL.md](THREAT_MODEL.md) → [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | ## Start here | Document | Purpose | |---|---| | [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) | Adoption dashboard — can I pin strict mode today? | +| [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | Authoritative R2 deep audit (supersedes day-to-day VISION_AUDIT) | +| [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md) | R2 sprint/PR execution program | | [RELEASE_AUDIT.md](RELEASE_AUDIT.md) | Engineering audit responses and metric provenance | | [STATUS.md](STATUS.md) | Capabilities, CLI surface, and trust model | | [EXPERIMENTAL_PATHS.md](EXPERIMENTAL_PATHS.md) | Honest limits for non-strict compiler/backend paths | @@ -35,6 +37,9 @@ Use this index as the canonical entry point. Each guide covers one topic; cross- | [EXTERNAL_VALIDATION.md](EXTERNAL_VALIDATION.md) | Weekly external validation matrix | | [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) | Immutable-pin consumer validation checklist (scaffolding) | | [FORMALPR_HOLDOUT_GOVERNANCE.md](FORMALPR_HOLDOUT_GOVERNANCE.md) | Private FormalPR-Holdout governance stub (no corpus) | +| [HOLDOUT_LABEL_SEPARATION.md](HOLDOUT_LABEL_SEPARATION.md) | Sprint 8 label-separated prediction/eval checklist | +| [SOURCE_PROFILE_HARDENING.md](SOURCE_PROFILE_HARDENING.md) | Sprint 6 source-profile hardening scaffolding | +| [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md) | Sprint 10 rc.1 / v1.3.0 publication gate | | [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) | Advisory→strict rollout on external OSS repos | | [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md) | In-repo pilot metrics and external pilot reporting | | [AGENT_REPAIR_LOOP.md](AGENT_REPAIR_LOOP.md) | Counterexample-to-repair workflow for MCP agents | diff --git a/docs/RELEASE.md b/docs/RELEASE.md index f88080d..8ce747d 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -1,6 +1,6 @@ # OVK Release -Maintainer guide for shipping Open Verification Kernel. Current readiness: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). Full code and artifact assessment: [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md). +Maintainer guide for shipping Open Verification Kernel. Current readiness: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). Authoritative audit: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Engineering program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). Historical: [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md). ## Current release candidate diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 341dc7a..9ff6e34 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,23 +1,26 @@ # OVK Roadmap -Current release: **v1.2.0**. What OVK can do today: [STATUS.md](STATUS.md). Adoption status: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). +Current product positioning: **`v1.3.0-rc.1` candidate** (typed control plane post-dates signed `v1.2.1`). What OVK can do today: [STATUS.md](STATUS.md). Adoption status: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). Authoritative program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). ## Release history | Version | Summary | Changelog | |---|---|---| +| v1.3.0-rc.1 (candidate) | Typed backend control plane; P0 trust PRs 1–9 in working tree; publication gates open | [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | +| v1.2.1 | Signed release on pre-control-plane commit; consumer pin baseline | [RELEASE_NOTES_v1.2.1.md](RELEASE_NOTES_v1.2.1.md) | | v1.2.0 | All five check types validated end-to-end; clearer GitHub Action outputs; example rollout workflows | [RELEASE_NOTES_v1.2.0.md](RELEASE_NOTES_v1.2.0.md) | | v1.1.0 | Realistic PR diff benchmark set; required native checker CI for OPA, Z3, CBMC, Cedar; external rollout guide | [RELEASE_NOTES_v1.1.0.md](RELEASE_NOTES_v1.1.0.md) | | v1.0.0 | Unified `ovk check`, five check types, ten backends, GitHub Action, benchmark suite | [RELEASE_NOTES_v1.0.0.md](RELEASE_NOTES_v1.0.0.md) | ## What we are working on next -1. **External repository pilots** — publish measured outcomes from real open-source adopters in [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md). -2. **Richer authorization checks** — more expressive Z3 obligations and smaller, clearer counterexamples. -3. **Community backends** — adapters and install docs beyond the current native checker matrix. -4. **CBMC diff depth** — richer extraction from real C PR hunks and repair-loop fixtures for CBMC counterexamples. +1. **Sprint 0 / attributable gates** — live CI, wheel smoke, Action dogfood, and workflow IDs on a non-`[skip ci]` SHA (P0 code PRs 1–9 already in working tree; see [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md)). +2. **Semantic template conformance v2** and source-profile hardening (Sprints 6–7). +3. **Consumer validation on rc.1** and label-separated holdout (Sprints 8–9). +4. **Attributable publication** of `v1.3.0-rc.1` then `v1.3.0` after the 18-condition gate (Sprint 10). ## Not planned as product promises - PyPI publication depends on maintainer release tagging (workflow is ready). - Optional native checkers (TLA+, Kani, Dafny, Verus, Lean, Alloy) remain non-blocking in CI until their harnesses mature. +- Re-attributing `v1.2.1` Sigstore/CI evidence to typed-control-plane commits. diff --git a/docs/VISION_AUDIT_2026-07-22.md b/docs/VISION_AUDIT_2026-07-22.md index 0bf35b5..b9fcd55 100644 --- a/docs/VISION_AUDIT_2026-07-22.md +++ b/docs/VISION_AUDIT_2026-07-22.md @@ -1,5 +1,7 @@ # OVK Vision Audit — 2026-07-22 +> **Historical.** For current release judgment and P0 trust gaps, use [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) and [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). This document predates the typed backend control plane and describes routing as advisory metadata. + ## Executive judgment Open Verification Kernel has achieved a credible **verification evidence product** for a bounded set of high-risk pull-request changes. It can infer five core checks from diffs, compile normalized lane inputs, execute deterministic or selected native checkers, aggregate evidence, produce conservative merge recommendations, render review output, and write hash-bound release bundles. From c9edd61d16f0e5c70feeb66206cfdeee070cc8c4 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:52:35 -0700 Subject: [PATCH 03/26] Bind CI-secrets and bridge material sizes to payloads. Stop deriving size_bytes from digest hex length so material references remain honest under size regression checks and shadow/compiler bridges. --- ovk/core/ci_secrets_compiler.py | 7 ++--- ovk/core/compiler_bridge.py | 10 +++---- ovk/core/shadow_obligation.py | 7 ++--- tests/test_material_reference_sizes.py | 40 ++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/ovk/core/ci_secrets_compiler.py b/ovk/core/ci_secrets_compiler.py index b0d41c9..79e4e13 100644 --- a/ovk/core/ci_secrets_compiler.py +++ b/ovk/core/ci_secrets_compiler.py @@ -14,11 +14,11 @@ ) from ovk.core.execution_models import ( AbstractionCoverage, - MaterialReference, VerificationObligation, compute_abstraction_digest, compute_obligation_id, ) +from ovk.core.materials import material_reference_from_payload from ovk.core.models import RiskSeverity, VerificationSubject COMPILER_ID = "ovk.ci_secrets.neutral.v1" @@ -97,12 +97,11 @@ def compile_ci_secrets_obligation( ) lane_input = data materials = [ - MaterialReference( + material_reference_from_payload( material_id="ci-secrets-input", kind="diff", uri="ovk-material:ci_secrets/input", - sha256=content_digest(data), - size_bytes=len(content_digest(data)), + payload=data, source_revision=head_sha, trusted=False, ) diff --git a/ovk/core/compiler_bridge.py b/ovk/core/compiler_bridge.py index f8956a4..bcd90ec 100644 --- a/ovk/core/compiler_bridge.py +++ b/ovk/core/compiler_bridge.py @@ -33,7 +33,7 @@ from ovk.compilers.infrastructure import compile_kubernetes_objects, compile_terraform_plan from ovk.compilers.infrastructure.ir import InfrastructureIR from ovk.core.execution_models import AbstractionCoverage, MaterialReference -from ovk.core.bundle import content_digest +from ovk.core.materials import material_reference_from_payload def coverage_policy_from_dict(policy: dict[str, Any] | None) -> CoveragePolicy: @@ -351,13 +351,11 @@ def material_refs_from_digest( source_revision: str | None, trusted: bool = False, ) -> MaterialReference: - digest = content_digest(payload) - return MaterialReference( + return material_reference_from_payload( material_id=material_id[:32], - kind=kind, # type: ignore[arg-type] + kind=kind, uri=uri, - sha256=digest, - size_bytes=len(digest), + payload=payload, source_revision=source_revision, trusted=trusted, ) diff --git a/ovk/core/shadow_obligation.py b/ovk/core/shadow_obligation.py index 4eeaefc..50f1099 100644 --- a/ovk/core/shadow_obligation.py +++ b/ovk/core/shadow_obligation.py @@ -7,11 +7,11 @@ from ovk.core.bundle import content_digest from ovk.core.execution_models import ( AbstractionCoverage, - MaterialReference, VerificationObligation, compute_abstraction_digest, compute_obligation_id, ) +from ovk.core.materials import material_reference_from_payload from ovk.core.models import RiskSeverity, VerificationSubject LANE_PROPERTY_KIND: dict[str, str] = { @@ -43,12 +43,11 @@ def build_shadow_obligation( ) -> VerificationObligation: """Construct a backend-neutral obligation for shadow control-plane execution.""" subject = VerificationSubject(repo=repo, head_sha=head_sha, base_sha=base_sha) - material = MaterialReference( + material = material_reference_from_payload( material_id=content_digest({"lane": lane, "input": data})[:32], kind="diff", uri=f"ovk-material:lane/{lane}", - sha256=content_digest(data), - size_bytes=len(content_digest(data)), + payload=data, source_revision=head_sha, trusted=False, ) diff --git a/tests/test_material_reference_sizes.py b/tests/test_material_reference_sizes.py index ffb39ca..27d1ce2 100644 --- a/tests/test_material_reference_sizes.py +++ b/tests/test_material_reference_sizes.py @@ -1,5 +1,8 @@ +from ovk.core.ci_secrets_compiler import compile_ci_secrets_obligation +from ovk.core.compiler_bridge import material_refs_from_digest from ovk.core.materials import canonical_material_bytes, material_reference_from_payload from ovk.core.self_protection_compiler import compile_self_protection_obligation +from ovk.core.shadow_obligation import build_shadow_obligation def test_material_reference_size_matches_canonical_payload_bytes() -> None: @@ -33,3 +36,40 @@ def test_self_protection_material_sizes_bind_each_payload() -> None: assert by_id["self-protection-before"].size_bytes == len(canonical_material_bytes(data["before"])) assert by_id["self-protection-after"].size_bytes == len(canonical_material_bytes(data["after"])) assert by_id["self-protection-input"].size_bytes == len(canonical_material_bytes(data)) + + +def test_ci_secrets_material_size_binds_payload_not_digest() -> None: + data = { + "workflows": { + "ci.yml": { + "on": "pull_request_target", + "jobs": {"build": {"runs-on": "ubuntu-latest", "steps": []}}, + } + } + } + obligation = compile_ci_secrets_obligation(data, repo="example/repo", head_sha="head") + material = obligation.materials[0] + assert material.size_bytes == len(canonical_material_bytes(data)) + assert material.size_bytes != len(material.sha256) + + +def test_shadow_and_bridge_material_sizes_bind_payload() -> None: + data = {"routes": [{"path": "/admin", "methods": ["GET"]}]} + shadow = build_shadow_obligation( + lane="authorization", + data=data, + repo="example/repo", + head_sha="head", + base_sha="base", + intent_id="admin-only-routes", + ) + assert shadow.materials[0].size_bytes == len(canonical_material_bytes(data)) + bridged = material_refs_from_digest( + material_id="bridge-material", + kind="diff", + uri="ovk-material:bridge", + payload=data, + source_revision="head", + ) + assert bridged.size_bytes == len(canonical_material_bytes(data)) + assert bridged.size_bytes != len(bridged.sha256) From df42f12a79d3e135f2ccf9bbed1703f57b156d97 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:52:35 -0700 Subject: [PATCH 04/26] Tighten LocalSubprocessWorker env allowlist and budgets. Inherit only allowlisted parent variables, always strip secrets, and reject non-positive wall-time budgets so workers fail closed instead of inheriting credentials or running unbounded. --- ovk/core/execution_budget.py | 42 +++++++++++++++--------- tests/test_cache_worker_control_plane.py | 24 +++++++++++++- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/ovk/core/execution_budget.py b/ovk/core/execution_budget.py index e91dd2b..f672237 100644 --- a/ovk/core/execution_budget.py +++ b/ovk/core/execution_budget.py @@ -117,8 +117,9 @@ def run( class LocalSubprocessWorker: """Local subprocess worker with timeout, cwd bound, and env allowlist. - Secret-bearing environment variables are never inherited. The parent - environment is otherwise preserved so native toolchains keep working in CI. + Child processes inherit only configured safe parent variables, plus any + explicit non-secret additions supplied by the caller. Secret-bearing + names are always stripped. Non-positive wall-time budgets are rejected. """ allowed_env_keys: frozenset[str] = field( @@ -152,6 +153,16 @@ def run( max_stdout_bytes: int = 1_000_000, max_stderr_bytes: int = 1_000_000, ) -> WorkerResult: + if timeout_seconds <= 0: + return WorkerResult( + exit_code=None, + timed_out=False, + stdout="", + stderr=f"non-positive wall-time budget rejected: {timeout_seconds}", + cwd=str(cwd.resolve()) if cwd else None, + command=tuple(command), + ) + cwd_resolved = cwd.resolve() if self.bound_roots: if not any(_is_relative_to(cwd_resolved, root.resolve()) for root in self.bound_roots): @@ -171,7 +182,7 @@ def run( cwd=str(cwd_resolved), env=child_env, capture_output=True, - timeout=timeout_seconds if timeout_seconds > 0 else None, + timeout=timeout_seconds, check=False, ) except subprocess.TimeoutExpired as exc: @@ -202,19 +213,17 @@ def run( ) def _build_env(self, extra: Mapping[str, str] | None) -> dict[str, str]: - """Build a child environment. + """Build a child environment from the configured allowlist only. - Default behavior inherits the parent environment except for known - secret-bearing keys. Callers that need a tighter sandbox can pass - ``extra`` with only the keys they need after constructing a worker - whose ``allowed_env_keys`` is interpreted as an optional soft guide - for documentation; secrets are always stripped. + Parent variables are inherited only when their names appear in + ``allowed_env_keys``. Explicit ``extra`` keys are merged unless they + match the secret denylist. Secret-bearing names are always removed. """ - baseline = { - key: value - for key, value in os.environ.items() - if key.upper() not in _SECRET_ENV_DENYLIST - } + allow = {key.upper() for key in self.allowed_env_keys} + baseline: dict[str, str] = {} + for key, value in os.environ.items(): + if key.upper() in allow and key.upper() not in _SECRET_ENV_DENYLIST: + baseline[key] = value if extra: for key, value in extra.items(): if key.upper() in _SECRET_ENV_DENYLIST: @@ -222,8 +231,11 @@ def _build_env(self, extra: Mapping[str, str] | None) -> dict[str, str]: baseline[key] = value for denied in _SECRET_ENV_DENYLIST: baseline.pop(denied, None) - # Also drop common case variants. baseline.pop(denied.lower(), None) + # Drop any case variant that might have been injected. + for existing in list(baseline): + if existing.upper() == denied.upper(): + baseline.pop(existing, None) return baseline diff --git a/tests/test_cache_worker_control_plane.py b/tests/test_cache_worker_control_plane.py index 187b7f9..78712c6 100644 --- a/tests/test_cache_worker_control_plane.py +++ b/tests/test_cache_worker_control_plane.py @@ -127,13 +127,20 @@ def test_worker_timeout_never_deterministic_pass(tmp_path: Path) -> None: def test_worker_env_allowlist_no_secret_inheritance(tmp_path: Path, monkeypatch) -> None: monkeypatch.setenv("GITHUB_TOKEN", "leak-me") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "also-leak") + monkeypatch.setenv("CUSTOM_UNKNOWN_CRED", "should-not-inherit") monkeypatch.setenv("PATH", os.environ.get("PATH", "")) worker = LocalSubprocessWorker(bound_roots=(tmp_path,)) result = worker.run( [ "python", "-c", - "import os; print(os.environ.get('GITHUB_TOKEN','')+os.environ.get('AWS_SECRET_ACCESS_KEY',''))", + ( + "import os; " + "print('TOK=' + os.environ.get('GITHUB_TOKEN','')); " + "print('AWS=' + os.environ.get('AWS_SECRET_ACCESS_KEY','')); " + "print('UNK=' + os.environ.get('CUSTOM_UNKNOWN_CRED','')); " + "print('SAFE=' + os.environ.get('OVK_WORKER_SAFE',''))" + ), ], cwd=tmp_path, env={"GITHUB_TOKEN": "should-not-pass", "OVK_WORKER_SAFE": "ok"}, @@ -143,6 +150,21 @@ def test_worker_env_allowlist_no_secret_inheritance(tmp_path: Path, monkeypatch) assert "leak-me" not in result.stdout assert "also-leak" not in result.stdout assert "should-not-pass" not in result.stdout + assert "should-not-inherit" not in result.stdout + assert "SAFE=ok" in result.stdout + + +def test_worker_rejects_non_positive_wall_budget(tmp_path: Path) -> None: + worker = LocalSubprocessWorker(bound_roots=(tmp_path,)) + result = worker.run( + ["python", "-c", "print('should-not-run')"], + cwd=tmp_path, + timeout_seconds=0, + ) + assert result.exit_code is None + assert result.timed_out is False + assert "non-positive" in result.stderr + assert "should-not-run" not in result.stdout def test_opa_runner_uses_worker_timeout(tmp_path: Path) -> None: From 9f535b60fe0104e26c105cf0e2fb639671c4e192 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:52:49 -0700 Subject: [PATCH 05/26] Isolate FormalPR-Holdout download tokens and require asset digests. Download and verify release assets in a token-bearing step, then evaluate without secrets so holdout aggregation cannot leak credentials or accept unverified corpora. --- .github/workflows/holdout-eval.yml | 61 +++++++-- docs/FORMALPR_HOLDOUT_GOVERNANCE.md | 4 +- scripts/run_formalpr_holdout.py | 172 ++++++++++++++++++++++++-- tests/test_formalpr_holdout_runner.py | 119 +++++++++++++++++- 4 files changed, 331 insertions(+), 25 deletions(-) diff --git a/.github/workflows/holdout-eval.yml b/.github/workflows/holdout-eval.yml index 6edf497..ec4c34b 100644 --- a/.github/workflows/holdout-eval.yml +++ b/.github/workflows/holdout-eval.yml @@ -3,6 +3,7 @@ name: FormalPR-Holdout eval # Optional gated job. Does not run on ordinary push/PR CI. # Requires repository secret HOLDOUT_DOWNLOAD_TOKEN with read access to # fraware/FormalPR-Holdout private release assets. +# Requires repository variable HOLDOUT_ASSET_SHA256 (immutable asset digest). on: workflow_dispatch: inputs: @@ -11,6 +12,11 @@ on: required: true default: v0.1.0-synthetic type: string + asset_sha256: + description: Immutable SHA-256 of the release asset (overrides repo variable) + required: false + default: "" + type: string predictions_artifact: description: Optional name of a workflow artifact containing predictions.json required: false @@ -28,13 +34,19 @@ jobs: id: gate env: HOLDOUT_DOWNLOAD_TOKEN: ${{ secrets.HOLDOUT_DOWNLOAD_TOKEN }} + HOLDOUT_ASSET_SHA256: ${{ inputs.asset_sha256 || vars.HOLDOUT_ASSET_SHA256 }} run: | if [ -z "$HOLDOUT_DOWNLOAD_TOKEN" ]; then echo "configured=false" >> "$GITHUB_OUTPUT" echo "FormalPR-Holdout eval skipped: HOLDOUT_DOWNLOAD_TOKEN not configured." echo "See docs/FORMALPR_HOLDOUT_GOVERNANCE.md" + elif [ -z "$HOLDOUT_ASSET_SHA256" ]; then + echo "configured=false" >> "$GITHUB_OUTPUT" + echo "FormalPR-Holdout eval skipped: HOLDOUT_ASSET_SHA256 not configured." + echo "Provide workflow input asset_sha256 or repository variable HOLDOUT_ASSET_SHA256." else echo "configured=true" >> "$GITHUB_OUTPUT" + echo "asset_sha256=$HOLDOUT_ASSET_SHA256" >> "$GITHUB_OUTPUT" fi - uses: actions/checkout@v4 @@ -76,12 +88,45 @@ jobs: fi test -f .verification/holdout-predictions.json - - name: Run holdout aggregate eval (fail closed) + - name: Download holdout asset (token only here) if: steps.gate.outputs.configured == 'true' + id: download env: HOLDOUT_DOWNLOAD_TOKEN: ${{ secrets.HOLDOUT_DOWNLOAD_TOKEN }} run: | + python - <<'PY' + import os + from pathlib import Path + from scripts.run_formalpr_holdout import download_release_asset, verify_asset_sha256 + + tag = "${{ inputs.holdout_tag }}" + asset_name = f"FormalPR-Holdout-{tag}.tar.gz" + dest = Path(".verification") / asset_name + dest.parent.mkdir(parents=True, exist_ok=True) + download_release_asset( + repo="fraware/FormalPR-Holdout", + tag=tag, + asset_name=asset_name, + dest=dest, + token=os.environ["HOLDOUT_DOWNLOAD_TOKEN"], + ) + verify_asset_sha256(dest, "${{ steps.gate.outputs.asset_sha256 }}") + print(f"downloaded and verified {dest}") + PY + echo "artifact=.verification/FormalPR-Holdout-${{ inputs.holdout_tag }}.tar.gz" >> "$GITHUB_OUTPUT" + + - name: Run holdout aggregate eval (fail closed, no tokens) + if: steps.gate.outputs.configured == 'true' + env: + # Explicitly unset download tokens for the evaluator process. + HOLDOUT_DOWNLOAD_TOKEN: "" + GITHUB_TOKEN: "" + GH_TOKEN: "" + run: | + unset HOLDOUT_DOWNLOAD_TOKEN GITHUB_TOKEN GH_TOKEN || true python scripts/run_formalpr_holdout.py \ + --artifact "${{ steps.download.outputs.artifact }}" \ + --asset-sha256 "${{ steps.gate.outputs.asset_sha256 }}" \ --tag "${{ inputs.holdout_tag }}" \ --predictions .verification/holdout-predictions.json \ --ovk-commit-sha "${GITHUB_SHA}" \ @@ -94,19 +139,9 @@ jobs: python - <<'PY' import json from pathlib import Path + from scripts.run_formalpr_holdout import validate_aggregate_schema payload = json.loads(Path(".verification/holdout-aggregate-metrics.json").read_text()) - assert payload["schema_version"] == "formalpr_holdout.aggregate_metrics.v1" - assert payload["leakage_guard"]["labels_emitted"] is False - assert payload["leakage_guard"]["case_ids_emitted"] is False - assert "lanes" in payload and payload["lanes"] - for lane, metrics in payload["lanes"].items(): - for key in ( - "precision", "recall", "false_positive_rate", "missed_detection_rate", - "unknown_rate", "invalid_input_rate", "abstention_appropriateness", - "coverage_completeness", "counterexample_correctness", - "selected_backend_execution_correctness", "runtime_ms", - ): - assert key in metrics, f"missing {key} in lane {lane}" + validate_aggregate_schema(payload) print("aggregate schema checks passed; labels not present") PY diff --git a/docs/FORMALPR_HOLDOUT_GOVERNANCE.md b/docs/FORMALPR_HOLDOUT_GOVERNANCE.md index 5309802..bd73665 100644 --- a/docs/FORMALPR_HOLDOUT_GOVERNANCE.md +++ b/docs/FORMALPR_HOLDOUT_GOVERNANCE.md @@ -35,7 +35,9 @@ Protected labels live only in private release assets (and gitignored local - Publishes **aggregate metrics only**; fails closed if labels or case ids would be printed. - Ordinary `CI` jobs do **not** checkout holdout labels. -Runner: `scripts/run_formalpr_holdout.py`. +Runner: `scripts/run_formalpr_holdout.py` (requires immutable `--asset-sha256`). + +Label-separated prediction/eval flow (Sprint 8): [HOLDOUT_LABEL_SEPARATION.md](HOLDOUT_LABEL_SEPARATION.md). ## What this is not diff --git a/scripts/run_formalpr_holdout.py b/scripts/run_formalpr_holdout.py index b59f40e..eb2f684 100644 --- a/scripts/run_formalpr_holdout.py +++ b/scripts/run_formalpr_holdout.py @@ -3,13 +3,22 @@ Fail-closed: never prints protected labels or case ids. Ordinary OVK CI should not invoke this without HOLDOUT_DOWNLOAD_TOKEN (or a pre-fetched artifact). + +Supply-chain requirements: +- immutable SHA-256 of the release asset must be supplied and verified; +- archive extraction is path-safe (no links, devices, FIFOs, or traversal); +- evaluate.py runs with an isolated env that excludes GitHub/holdout tokens; +- aggregate output is schema-validated and leakage-guarded. """ from __future__ import annotations import argparse +import hashlib import json import os +import re +import stat import subprocess import sys import tarfile @@ -17,6 +26,7 @@ import urllib.error import urllib.request from pathlib import Path +from typing import Any FORBIDDEN_SUBSTRINGS = ( @@ -34,12 +44,33 @@ "syn-unknown-surface-01", ) +_TOKEN_ENV_KEYS = frozenset( + { + "HOLDOUT_DOWNLOAD_TOKEN", + "GITHUB_TOKEN", + "GH_TOKEN", + "ACTIONS_RUNTIME_TOKEN", + } +) + +_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") + +_REQUIRED_AGGREGATE_KEYS = ( + "schema_version", + "benchmark", + "holdout_release_tag", + "ovk_commit_sha", + "cases_scored", + "lanes", + "leakage_guard", +) + def _fail(msg: str) -> None: raise SystemExit(f"fail-closed: {msg}") -def assert_aggregate_safe(payload: dict) -> None: +def assert_aggregate_safe(payload: dict[str, Any]) -> None: text = json.dumps(payload) for token in FORBIDDEN_SUBSTRINGS: if token in text: @@ -55,6 +86,37 @@ def assert_aggregate_safe(payload: dict) -> None: _fail("refusing pass-rate-only payload") +def validate_aggregate_schema(payload: dict[str, Any]) -> None: + """Fail-closed structural validation for holdout aggregate metrics.""" + for key in _REQUIRED_AGGREGATE_KEYS: + if key not in payload: + _fail(f"aggregate missing required key {key!r}") + if payload.get("schema_version") != "formalpr_holdout.aggregate_metrics.v1": + _fail("unexpected aggregate schema_version") + if payload.get("benchmark") != "FormalPR-Holdout": + _fail("unexpected aggregate benchmark") + lanes = payload.get("lanes") + if not isinstance(lanes, dict) or not lanes: + _fail("aggregate lanes must be a non-empty object") + for lane, metrics in lanes.items(): + if not isinstance(metrics, dict): + _fail(f"lane {lane!r} metrics must be an object") + for key in ( + "precision", + "recall", + "false_positive_rate", + "missed_detection_rate", + "unknown_rate", + "coverage_completeness", + "counterexample_correctness", + "selected_backend_execution_correctness", + "runtime_ms", + ): + if key not in metrics: + _fail(f"missing {key} in lane {lane}") + assert_aggregate_safe(payload) + + def download_release_asset( *, repo: str, @@ -92,20 +154,92 @@ def download_release_asset( return dest +def verify_asset_sha256(path: Path, expected_sha256: str) -> str: + if not _SHA256_RE.match(expected_sha256): + _fail("asset SHA-256 must be a 64-character hex digest") + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if digest.lower() != expected_sha256.lower(): + _fail("asset SHA-256 mismatch") + return digest + + +def _is_unsafe_tar_member(member: tarfile.TarInfo) -> str | None: + name = member.name.replace("\\", "/") + if name.startswith("/") or name.startswith("..") or "/../" in f"/{name}/": + return f"unsafe path {member.name!r}" + if member.issym() or member.islnk(): + return f"link member forbidden: {member.name!r}" + if member.isdev() or member.isfifo() or member.ischr() or member.isblk(): + return f"special file forbidden: {member.name!r}" + # Reject unusual types beyond regular files and directories. + if not (member.isfile() or member.isdir()): + return f"unsupported member type: {member.name!r}" + return None + + def extract_tarball(tarball: Path, dest: Path) -> Path: + """Path-safe extraction: no links/devices/FIFOs/traversal; no filter=data reliance.""" dest.mkdir(parents=True, exist_ok=True) + dest_resolved = dest.resolve() with tarfile.open(tarball, "r:gz") as tar: - # Python 3.12+ supports filter=; keep compatible call. - try: - tar.extractall(dest, filter="data") - except TypeError: - tar.extractall(dest) + for member in tar.getmembers(): + reason = _is_unsafe_tar_member(member) + if reason: + _fail(reason) + target = (dest / member.name).resolve() + if not str(target).startswith(str(dest_resolved) + os.sep) and target != dest_resolved: + _fail(f"path traversal forbidden: {member.name!r}") + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + continue + target.parent.mkdir(parents=True, exist_ok=True) + extracted = tar.extractfile(member) + if extracted is None: + _fail(f"cannot extract member {member.name!r}") + with extracted, open(target, "wb") as out: + out.write(extracted.read()) + # Ensure we did not create a link somehow. + if target.is_symlink() or not target.is_file(): + _fail(f"extracted path is not a regular file: {member.name!r}") + mode = target.stat().st_mode + if stat.S_ISLNK(mode) or stat.S_ISCHR(mode) or stat.S_ISBLK(mode) or stat.S_ISFIFO(mode): + _fail(f"special file after extract: {member.name!r}") roots = [p for p in dest.iterdir() if p.is_dir()] if len(roots) != 1: _fail(f"expected one release root, found {len(roots)}") return roots[0] +def _isolated_eval_env() -> dict[str, str]: + """Build an evaluator environment without download/GitHub tokens.""" + allow = { + "PATH", + "PATHEXT", + "SYSTEMROOT", + "TEMP", + "TMP", + "TMPDIR", + "HOME", + "USERPROFILE", + "LANG", + "LC_ALL", + "PYTHONPATH", + "VIRTUAL_ENV", + "PYTHONNOUSERSITE", + "PYTHONUTF8", + "PYTHONIOENCODING", + } + env = {k: v for k, v in os.environ.items() if k.upper() in allow} + env["PYTHONNOUSERSITE"] = "1" + for key in list(env): + if key.upper() in _TOKEN_ENV_KEYS: + env.pop(key, None) + # Explicitly ensure tokens are absent even if allowlist was widened later. + for denied in _TOKEN_ENV_KEYS: + env.pop(denied, None) + return env + + def run_harness( *, release_root: Path, @@ -114,7 +248,7 @@ def run_harness( ovk_sha: str, verified_sha: str | None, output: Path, -) -> dict: +) -> dict[str, Any]: evaluate = release_root / "harness" / "evaluate.py" if not evaluate.is_file(): _fail("release artifact missing harness/evaluate.py") @@ -148,12 +282,19 @@ def run_harness( if verified_sha: cmd.extend(["--verified-source-sha", verified_sha]) # Do not pass --print-aggregates; we load the file and re-sanitize. - proc = subprocess.run(cmd, capture_output=True, text=True, check=False) + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + check=False, + env=_isolated_eval_env(), + cwd=str(release_root), + ) if proc.returncode != 0: # Avoid echoing stderr if it might contain paths with case ids — redact. _fail(f"evaluate.py exited {proc.returncode}") payload = json.loads(output.read_text(encoding="utf-8")) - assert_aggregate_safe(payload) + validate_aggregate_schema(payload) return payload @@ -172,6 +313,11 @@ def main(argv: list[str] | None = None) -> int: default=None, help="Use a pre-downloaded tarball instead of GitHub API download", ) + parser.add_argument( + "--asset-sha256", + required=True, + help="Immutable SHA-256 hex digest of the release asset (required)", + ) parser.add_argument("--predictions", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--ovk-commit-sha", required=True) @@ -205,7 +351,13 @@ def main(argv: list[str] | None = None) -> int: dest=tmp_path / asset_name, token=token, ) + verify_asset_sha256(tarball, args.asset_sha256) release_root = extract_tarball(tarball, tmp_path / "extract") + # Predictions must be label-free before the evaluator sees them. + pred_payload = json.loads(args.predictions.read_text(encoding="utf-8")) + from scripts.digest_holdout_predictions import assert_predictions_label_free + + assert_predictions_label_free(pred_payload) payload = run_harness( release_root=release_root, predictions=args.predictions, @@ -223,7 +375,7 @@ def main(argv: list[str] | None = None) -> int: ) print(summary) if args.print_aggregates: - assert_aggregate_safe(payload) + validate_aggregate_schema(payload) print(json.dumps(payload, indent=2)) return 0 diff --git a/tests/test_formalpr_holdout_runner.py b/tests/test_formalpr_holdout_runner.py index 147828a..8058431 100644 --- a/tests/test_formalpr_holdout_runner.py +++ b/tests/test_formalpr_holdout_runner.py @@ -2,12 +2,21 @@ from __future__ import annotations +import hashlib +import io import json +import tarfile from pathlib import Path import pytest -from scripts.run_formalpr_holdout import assert_aggregate_safe +from scripts.run_formalpr_holdout import ( + assert_aggregate_safe, + extract_tarball, + validate_aggregate_schema, + verify_asset_sha256, + _isolated_eval_env, +) def _valid_aggregate() -> dict: @@ -49,6 +58,10 @@ def test_assert_aggregate_safe_accepts_valid_payload() -> None: assert_aggregate_safe(_valid_aggregate()) +def test_validate_aggregate_schema_accepts_valid_payload() -> None: + validate_aggregate_schema(_valid_aggregate()) + + def test_assert_aggregate_safe_rejects_case_id_leak() -> None: payload = _valid_aggregate() payload["notes"] = "syn-auth-bypass-01" @@ -63,6 +76,107 @@ def test_assert_aggregate_safe_rejects_label_flag() -> None: assert_aggregate_safe(payload) +def test_validate_aggregate_schema_rejects_missing_lane_metric() -> None: + payload = _valid_aggregate() + del payload["lanes"]["authorization"]["precision"] + with pytest.raises(SystemExit, match="fail-closed"): + validate_aggregate_schema(payload) + + +def test_verify_asset_sha256_accepts_and_rejects(tmp_path: Path) -> None: + asset = tmp_path / "asset.tar.gz" + asset.write_bytes(b"holdout-bytes") + digest = hashlib.sha256(asset.read_bytes()).hexdigest() + assert verify_asset_sha256(asset, digest) == digest + with pytest.raises(SystemExit, match="SHA-256 mismatch"): + verify_asset_sha256(asset, "0" * 64) + with pytest.raises(SystemExit, match="64-character"): + verify_asset_sha256(asset, "not-a-digest") + + +def test_extract_tarball_rejects_symlink(tmp_path: Path) -> None: + tarball = tmp_path / "bad.tar.gz" + with tarfile.open(tarball, "w:gz") as tar: + info = tarfile.TarInfo(name="root/link") + info.type = tarfile.SYMTYPE + info.linkname = "../outside" + tar.addfile(info) + with pytest.raises(SystemExit, match="link member forbidden"): + extract_tarball(tarball, tmp_path / "out") + + +def test_extract_tarball_rejects_traversal(tmp_path: Path) -> None: + tarball = tmp_path / "trav.tar.gz" + with tarfile.open(tarball, "w:gz") as tar: + data = b"x" + info = tarfile.TarInfo(name="../evil.txt") + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + with pytest.raises(SystemExit, match="unsafe path|path traversal"): + extract_tarball(tarball, tmp_path / "out") + + +def test_extract_tarball_accepts_regular_tree(tmp_path: Path) -> None: + tarball = tmp_path / "good.tar.gz" + with tarfile.open(tarball, "w:gz") as tar: + dir_info = tarfile.TarInfo(name="release-root") + dir_info.type = tarfile.DIRTYPE + tar.addfile(dir_info) + payload = b'{"ok": true}' + file_info = tarfile.TarInfo(name="release-root/readme.json") + file_info.size = len(payload) + tar.addfile(file_info, io.BytesIO(payload)) + root = extract_tarball(tarball, tmp_path / "out") + assert root.name == "release-root" + assert (root / "readme.json").read_bytes() == payload + + +def test_isolated_eval_env_strips_tokens(monkeypatch) -> None: + monkeypatch.setenv("HOLDOUT_DOWNLOAD_TOKEN", "secret-holdout") + monkeypatch.setenv("GITHUB_TOKEN", "secret-gh") + monkeypatch.setenv("PATH", "/usr/bin") + env = _isolated_eval_env() + assert "HOLDOUT_DOWNLOAD_TOKEN" not in env + assert "GITHUB_TOKEN" not in env + assert env.get("PATH") == "/usr/bin" + + +def test_predictions_digest_rejects_embedded_labels(tmp_path: Path) -> None: + from scripts.digest_holdout_predictions import assert_predictions_label_free, digest_predictions_file + + clean = {"predictions": [{"case_id": "case-1", "status": "fail", "merge_recommendation": "block"}]} + assert_predictions_label_free(clean) + path = tmp_path / "predictions.json" + path.write_text(json.dumps(clean), encoding="utf-8") + record = digest_predictions_file(path) + assert record["label_free"] is True + assert len(record["sha256"]) == 64 + + dirty = { + "predictions": [ + { + "case_id": "case-1", + "status": "fail", + "expected_status": "fail", + } + ] + } + with pytest.raises(SystemExit, match="fail-closed"): + assert_predictions_label_free(dirty) + + +def test_predictions_digest_rejects_forbidden_substring(tmp_path: Path) -> None: + from scripts.digest_holdout_predictions import digest_predictions_file + + path = tmp_path / "predictions.json" + path.write_text( + json.dumps({"predictions": [{"case_id": "x", "note": "ground_truth_class leaked"}]}), + encoding="utf-8", + ) + with pytest.raises(SystemExit, match="fail-closed"): + digest_predictions_file(path) + + def test_run_formalpr_holdout_against_local_artifact(tmp_path: Path) -> None: """End-to-end against a sibling-packaged release if present.""" holdout_root = Path(__file__).resolve().parents[1].parent / "FormalPR-Holdout" @@ -99,10 +213,13 @@ def test_run_formalpr_holdout_against_local_artifact(tmp_path: Path) -> None: pred_path = tmp_path / "predictions.json" pred_path.write_text(json.dumps(preds), encoding="utf-8") out = tmp_path / "agg.json" + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() rc = runner.main( [ "--artifact", str(artifact), + "--asset-sha256", + digest, "--tag", "v0.1.0-synthetic", "--predictions", From 397d020e5f16f92a6f4d2be8a0943d3154662af1 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:52:49 -0700 Subject: [PATCH 06/26] Separate benchmark_source_sha from verified_source_sha. Badge and summary metrics may cite the measured commit without labeling skip-ci or incomplete workflow SHAs as verified source provenance. --- ovk/core/verified_source.py | 61 +++++++++++++++++++-------- schemas/adoption.summary.schema.json | 7 +++- scripts/render_bench_badge.py | 62 ++++++++++++++++++++-------- tests/test_bench_badge.py | 23 ++++++++--- tests/test_verified_source.py | 23 ++++++++--- 5 files changed, 130 insertions(+), 46 deletions(-) diff --git a/ovk/core/verified_source.py b/ovk/core/verified_source.py index 3a3d0db..951f662 100644 --- a/ovk/core/verified_source.py +++ b/ovk/core/verified_source.py @@ -1,8 +1,11 @@ -"""Resolve the immutable commit that produced verification metrics. +"""Resolve benchmark vs verified source SHAs for metric provenance. -Badge and summary commits may use ``[skip ci]`` and must not be confused with -the commit that actually ran FormalPR-Bench / pilot metrics. Prefer an explicit -override, then GitHub Actions SHA, then local ``git rev-parse HEAD``. +``benchmark_source_sha`` is the commit whose FormalPR-Bench (or badge) artifacts +were measured. ``verified_source_sha`` is only set when a complete observed +required-workflow set is attested (explicit override or ``OVK_VERIFIED_SOURCE_SHA``). + +Badge-only / ``[skip ci]`` commits must never be labeled verified merely because +``GITHUB_SHA`` or ``git HEAD`` is available. """ from __future__ import annotations @@ -12,19 +15,7 @@ from pathlib import Path -def resolve_verified_source_sha( - *, - explicit: str | None = None, - repo_root: Path | None = None, -) -> str | None: - """Return a full git commit SHA for metric provenance, or None if unknown.""" - if explicit: - value = explicit.strip() - return value or None - for key in ("OVK_VERIFIED_SOURCE_SHA", "GITHUB_SHA"): - value = (os.environ.get(key) or "").strip() - if value: - return value +def _git_head(repo_root: Path | None) -> str | None: root = repo_root or Path(__file__).resolve().parents[1] try: completed = subprocess.run( @@ -41,3 +32,39 @@ def resolve_verified_source_sha( return None value = completed.stdout.strip() return value or None + + +def resolve_benchmark_source_sha( + *, + explicit: str | None = None, + repo_root: Path | None = None, +) -> str | None: + """Return the commit SHA that produced benchmark/badge artifacts, if known.""" + if explicit: + value = explicit.strip() + return value or None + for key in ("OVK_BENCHMARK_SOURCE_SHA", "GITHUB_SHA"): + value = (os.environ.get(key) or "").strip() + if value: + return value + return _git_head(repo_root) + + +def resolve_verified_source_sha( + *, + explicit: str | None = None, + repo_root: Path | None = None, +) -> str | None: + """Return a verified-source SHA only when explicitly attested. + + Does not fall back to ``GITHUB_SHA`` or ``git HEAD``. Those identify the + current checkout / workflow run and belong on ``benchmark_source_sha`` unless + maintainers also set ``OVK_VERIFIED_SOURCE_SHA`` after observing the full + required-workflow set. + """ + del repo_root # retained for API compatibility; verified SHA is never inferred from HEAD + if explicit: + value = explicit.strip() + return value or None + value = (os.environ.get("OVK_VERIFIED_SOURCE_SHA") or "").strip() + return value or None diff --git a/schemas/adoption.summary.schema.json b/schemas/adoption.summary.schema.json index 253d0e9..f74b72c 100644 --- a/schemas/adoption.summary.schema.json +++ b/schemas/adoption.summary.schema.json @@ -18,7 +18,12 @@ "verified_source_sha": { "type": "string", "minLength": 7, - "description": "Git commit that produced the metrics; not a later [skip ci] badge commit." + "description": "Git commit with a complete observed required-workflow set; never a [skip ci] badge-only commit." + }, + "benchmark_source_sha": { + "type": "string", + "minLength": 7, + "description": "Git commit measured by FormalPR-Bench / badge artifacts." }, "formal_pr_bench": { "type": "object", diff --git a/scripts/render_bench_badge.py b/scripts/render_bench_badge.py index 491612e..d6ec933 100644 --- a/scripts/render_bench_badge.py +++ b/scripts/render_bench_badge.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any -from ovk.core.verified_source import resolve_verified_source_sha +from ovk.core.verified_source import resolve_benchmark_source_sha, resolve_verified_source_sha ROOT = Path(__file__).resolve().parents[1] DEFAULT_LEADERBOARD = ROOT / ".verification" / "formal-pr-bench-leaderboard.json" @@ -30,38 +30,45 @@ def badge_color(cases_passed: int, cases_total: int) -> str: def render_badge( leaderboard: dict[str, Any], *, + benchmark_source_sha: str | None = None, verified_source_sha: str | None = None, ) -> dict[str, Any]: """Build shields.io endpoint badge payload. - ``verified_source_sha`` records the commit that produced the leaderboard, - not a later ``[skip ci]`` badge-only commit. + ``benchmark_source_sha`` is the commit measured by FormalPR-Bench. + ``verified_source_sha`` is set only when a complete required-workflow set was + observed for that source; badge-only commits must not be labeled verified. """ summary = leaderboard.get("summary", {}) cases_total = int(summary.get("cases_total", 0)) cases_passed = int(summary.get("cases_passed", 0)) rate = (cases_passed / cases_total * 100.0) if cases_total else 0.0 - sha = verified_source_sha or resolve_verified_source_sha() + bench_sha = benchmark_source_sha or resolve_benchmark_source_sha() + verified_sha = verified_source_sha or resolve_verified_source_sha() payload: dict[str, Any] = { "schemaVersion": 1, "label": "FormalPR-Bench", "message": f"{cases_passed}/{cases_total} ({rate:.0f}%)", "color": badge_color(cases_passed, cases_total), } - if sha: - payload["verified_source_sha"] = sha + if bench_sha: + payload["benchmark_source_sha"] = bench_sha + if verified_sha: + payload["verified_source_sha"] = verified_sha return payload def render_summary( leaderboard: dict[str, Any], *, + benchmark_source_sha: str | None = None, verified_source_sha: str | None = None, ) -> dict[str, Any]: """Build trimmed public summary for docs and README links.""" summary = leaderboard.get("summary", {}) timing = leaderboard.get("timing_ms", {}) - sha = verified_source_sha or resolve_verified_source_sha() + bench_sha = benchmark_source_sha or resolve_benchmark_source_sha() + verified_sha = verified_source_sha or resolve_verified_source_sha() payload: dict[str, Any] = { "schema_version": "formal_pr_bench.summary.v1", "generated_from": leaderboard.get("schema_version", "formal_pr_bench.leaderboard.v1"), @@ -86,12 +93,15 @@ def render_summary( "max": timing.get("max"), }, } - if sha: - payload["verified_source_sha"] = sha - payload["provenance_note"] = ( - "Cite verified_source_sha (and its CI run) for health claims; " - "do not treat a later [skip ci] badge commit as the verified source." - ) + if bench_sha: + payload["benchmark_source_sha"] = bench_sha + if verified_sha: + payload["verified_source_sha"] = verified_sha + payload["provenance_note"] = ( + "Cite benchmark_source_sha for FormalPR-Bench measurement identity. " + "Cite verified_source_sha only when a complete required-workflow set was " + "observed for that commit; do not treat a later [skip ci] badge commit as verified." + ) return payload @@ -99,13 +109,23 @@ def write_outputs( leaderboard_path: Path, *, dry_run: bool = False, + benchmark_source_sha: str | None = None, verified_source_sha: str | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: """Read leaderboard and write badge + summary files.""" leaderboard = json.loads(leaderboard_path.read_text(encoding="utf-8")) - sha = verified_source_sha or resolve_verified_source_sha() - badge = render_badge(leaderboard, verified_source_sha=sha) - summary = render_summary(leaderboard, verified_source_sha=sha) + bench_sha = benchmark_source_sha or resolve_benchmark_source_sha() + verified_sha = verified_source_sha or resolve_verified_source_sha() + badge = render_badge( + leaderboard, + benchmark_source_sha=bench_sha, + verified_source_sha=verified_sha, + ) + summary = render_summary( + leaderboard, + benchmark_source_sha=bench_sha, + verified_source_sha=verified_sha, + ) if not dry_run: BADGE_PATH.parent.mkdir(parents=True, exist_ok=True) BADGE_PATH.write_text(json.dumps(badge, indent=2) + "\n", encoding="utf-8") @@ -121,10 +141,15 @@ def main() -> int: default=DEFAULT_LEADERBOARD, help="Path to formal-pr-bench-leaderboard.json", ) + parser.add_argument( + "--benchmark-source-sha", + default=None, + help="Commit measured by FormalPR-Bench (defaults to GITHUB_SHA / git HEAD)", + ) parser.add_argument( "--verified-source-sha", default=None, - help="Commit that produced the leaderboard (defaults to GITHUB_SHA / git HEAD)", + help="Commit with a complete observed required-workflow set (optional)", ) parser.add_argument("--dry-run", action="store_true", help="Compute outputs without writing files") args = parser.parse_args() @@ -134,12 +159,15 @@ def main() -> int: badge, summary = write_outputs( args.leaderboard, dry_run=args.dry_run, + benchmark_source_sha=args.benchmark_source_sha, verified_source_sha=args.verified_source_sha, ) if args.dry_run: print(json.dumps({"badge": badge, "summary": summary}, indent=2)) else: print(f"wrote {BADGE_PATH.relative_to(ROOT)} and {SUMMARY_PATH.relative_to(ROOT)}") + if summary.get("benchmark_source_sha"): + print(f"benchmark_source_sha={summary['benchmark_source_sha']}") if summary.get("verified_source_sha"): print(f"verified_source_sha={summary['verified_source_sha']}") return 0 diff --git a/tests/test_bench_badge.py b/tests/test_bench_badge.py index 6db7382..149df09 100644 --- a/tests/test_bench_badge.py +++ b/tests/test_bench_badge.py @@ -18,12 +18,17 @@ def test_render_badge_shape() -> None: "summary": {"cases_total": 100, "cases_passed": 100}, "timing_ms": {"p50": 1.0, "p95": 2.0, "max": 3.0}, } - badge = render_badge(leaderboard, verified_source_sha="abc1234deadbeef") + badge = render_badge( + leaderboard, + benchmark_source_sha="abc1234deadbeef", + verified_source_sha=None, + ) assert badge["schemaVersion"] == 1 assert badge["label"] == "FormalPR-Bench" assert "100/100" in badge["message"] assert badge["color"] == "brightgreen" - assert badge["verified_source_sha"] == "abc1234deadbeef" + assert badge["benchmark_source_sha"] == "abc1234deadbeef" + assert "verified_source_sha" not in badge def test_render_summary_includes_dimensions() -> None: @@ -37,12 +42,18 @@ def test_render_summary_includes_dimensions() -> None: }, "timing_ms": {"p50": 1.0, "p95": 2.0, "max": 3.0}, } - summary = render_summary(leaderboard, verified_source_sha="sha-source") + summary = render_summary( + leaderboard, + benchmark_source_sha="sha-bench", + verified_source_sha="sha-verified", + ) assert summary["schema_version"] == "formal_pr_bench.summary.v1" assert summary["cases_passed"] == 9 assert summary["timing_ms"]["p95"] == 2.0 assert "lane" in summary["by_category"] - assert summary["verified_source_sha"] == "sha-source" + assert summary["benchmark_source_sha"] == "sha-bench" + assert summary["verified_source_sha"] == "sha-verified" + assert "benchmark_source_sha" in summary["provenance_note"] assert "skip ci" in summary["provenance_note"] @@ -52,5 +63,7 @@ def test_render_summary_includes_real_diff_recall() -> None: "summary": {"cases_total": 10, "cases_passed": 9, "real_diff_recall": 0.95}, "timing_ms": {"p50": 1.0, "p95": 2.0, "max": 3.0}, } - summary = render_summary(leaderboard, verified_source_sha="x" * 40) + summary = render_summary(leaderboard, benchmark_source_sha="x" * 40) assert summary["real_diff_recall"] == 0.95 + assert summary["benchmark_source_sha"] == "x" * 40 + assert "verified_source_sha" not in summary diff --git a/tests/test_verified_source.py b/tests/test_verified_source.py index 1d2ea92..21e6d6e 100644 --- a/tests/test_verified_source.py +++ b/tests/test_verified_source.py @@ -1,23 +1,34 @@ -"""Tests for verified_source_sha resolution.""" +"""Tests for verified_source_sha / benchmark_source_sha resolution.""" from __future__ import annotations -from ovk.core.verified_source import resolve_verified_source_sha +from ovk.core.verified_source import resolve_benchmark_source_sha, resolve_verified_source_sha -def test_explicit_sha_wins(monkeypatch) -> None: +def test_explicit_verified_sha_wins(monkeypatch) -> None: monkeypatch.setenv("GITHUB_SHA", "github-sha") monkeypatch.setenv("OVK_VERIFIED_SOURCE_SHA", "env-sha") assert resolve_verified_source_sha(explicit="explicit-sha") == "explicit-sha" -def test_ovk_env_beats_github(monkeypatch) -> None: +def test_ovk_env_is_verified(monkeypatch) -> None: monkeypatch.setenv("GITHUB_SHA", "github-sha") monkeypatch.setenv("OVK_VERIFIED_SOURCE_SHA", "env-sha") assert resolve_verified_source_sha() == "env-sha" -def test_github_sha_used_when_no_override(monkeypatch) -> None: +def test_github_sha_is_not_verified(monkeypatch) -> None: monkeypatch.delenv("OVK_VERIFIED_SOURCE_SHA", raising=False) monkeypatch.setenv("GITHUB_SHA", "github-sha-only") - assert resolve_verified_source_sha() == "github-sha-only" + assert resolve_verified_source_sha() is None + + +def test_github_sha_is_benchmark_source(monkeypatch) -> None: + monkeypatch.delenv("OVK_BENCHMARK_SOURCE_SHA", raising=False) + monkeypatch.setenv("GITHUB_SHA", "github-sha-only") + assert resolve_benchmark_source_sha() == "github-sha-only" + + +def test_explicit_benchmark_sha_wins(monkeypatch) -> None: + monkeypatch.setenv("GITHUB_SHA", "github-sha") + assert resolve_benchmark_source_sha(explicit="bench-sha") == "bench-sha" From 331004b80d735cb641fed5efc82fc20f9086b811 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:52:50 -0700 Subject: [PATCH 07/26] Exclude timing from deterministic attempt identity. Drop duration_ms from attempt digests and extend fallback/cache model types so equivalent runs share stable IDs while later PRs can carry provenance-preserving cache and cause-specific fallback policy. --- ovk/core/execution_models.py | 43 +++++++++++++++++++++++++++++++--- tests/test_execution_models.py | 20 +++++++++++++++- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/ovk/core/execution_models.py b/ovk/core/execution_models.py index 5170bf4..ea1384b 100644 --- a/ovk/core/execution_models.py +++ b/ovk/core/execution_models.py @@ -214,9 +214,21 @@ class FallbackPolicy(BaseModel): allow_fallback: bool = False fallback_backends: list[str] = Field(default_factory=list) + acceptable_fallback_guarantees: list[str] = Field(default_factory=list) on_timeout: FallbackOutcome = "unknown" on_tool_unavailable: FallbackOutcome = "unknown" on_invalid_output: FallbackOutcome = "unknown" + on_resource_exhausted: FallbackOutcome = "unknown" + + def outcome_for_termination(self, termination: TerminationKind | str) -> FallbackOutcome: + """Map a termination kind to the configured fallback outcome.""" + mapping: dict[str, FallbackOutcome] = { + "timeout": self.on_timeout, + "tool_unavailable": self.on_tool_unavailable, + "invalid_output": self.on_invalid_output, + "resource_exhausted": self.on_resource_exhausted, + } + return mapping.get(str(termination), "unknown") class BackendCapabilityAssessment(BaseModel): @@ -429,6 +441,25 @@ class NormalizedBackendResult(BaseModel): generated_artifacts: list[dict[str, Any]] = Field(default_factory=list) +class CachedBackendExecution(BaseModel): + """Provenance-preserving cache payload for a backend execution (ovk.cache.v3). + + Cache hits must replay this record rather than synthesizing a new attempt or + re-inferring ``native_execution`` from current tool availability. + """ + + schema_version: Literal["ovk.cache.v3"] = "ovk.cache.v3" + attempt: ExecutionAttempt + native_execution: bool + tool_version: str | None = None + tool_digest: str | None = None + termination: TerminationKind + exit_code: int | None = None + raw_result_digest: str | None = None + environment_fingerprint: str + normalized_result: NormalizedBackendResult + + class ObligationExecutionRecord(BaseModel): """Full typed record of compiling, routing, and executing one obligation.""" @@ -441,6 +472,9 @@ class ObligationExecutionRecord(BaseModel): merge_recommendation: MergeRecommendation aggregation_reason: str open_obligations: list[dict[str, Any]] = Field(default_factory=list) + fallback_used: bool = False + fallback_accepted: bool = False + fallback_cause: str | None = None # --------------------------------------------------------------------------- @@ -557,15 +591,18 @@ def compute_backend_obligation_id( def attempt_digest_input(attempt: ExecutionAttempt | dict[str, Any]) -> dict[str, Any]: - """Canonical digest payload for an execution attempt (excludes ``attempt_id`` and wall-clock times). + """Canonical digest payload for an execution attempt (excludes identity and timing). - Timestamps are excluded so identical executions remain content-addressable when - wall-clock values differ; ``duration_ms`` and termination/output digests remain. + ``attempt_id``, wall-clock timestamps, and ``duration_ms`` are excluded so + otherwise equivalent executions remain content-addressable across sequential, + parallel, cached, and uncached runs. Duration remains observational metadata + on the attempt object itself. """ data = dict(_dump_json(attempt)) data.pop("attempt_id", None) data.pop("started_at", None) data.pop("finished_at", None) + data.pop("duration_ms", None) return data diff --git a/tests/test_execution_models.py b/tests/test_execution_models.py index 08e26ff..f4f23ba 100644 --- a/tests/test_execution_models.py +++ b/tests/test_execution_models.py @@ -367,12 +367,30 @@ def test_attempt_id_excludes_wall_clock_timestamps() -> None: assert "attempt_id" not in digest_input assert "started_at" not in digest_input assert "finished_at" not in digest_input + assert "duration_ms" not in digest_input shifted = attempt.model_copy( - update={"started_at": "2099-01-01T00:00:00Z", "finished_at": "2099-01-01T00:00:01Z"} + update={ + "started_at": "2099-01-01T00:00:00Z", + "finished_at": "2099-01-01T00:00:01Z", + "duration_ms": 99999.0, + } ) assert compute_attempt_id(shifted) == compute_attempt_id(attempt) +def test_attempt_id_stable_across_duration_jitter() -> None: + """Equivalent semantic attempts must share IDs despite timing jitter.""" + base = _attempt("bo-stable") + variants = [ + base.model_copy(update={"duration_ms": 1.0}), + base.model_copy(update={"duration_ms": 50.5}), + base.model_copy(update={"duration_ms": 1200.0, "started_at": "2026-07-01T00:00:00Z"}), + base.model_copy(update={"duration_ms": 0.0}), + ] + ids = {compute_attempt_id(item) for item in [base, *variants]} + assert len(ids) == 1 + + def test_attempt_id_changes_when_termination_changes() -> None: attempt = _attempt("bo-1") other = attempt.model_copy(update={"termination": "timeout", "exit_code": None}) From 12aebdabb23292611938a3507b46cab30aaf001e Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:52:50 -0700 Subject: [PATCH 08/26] Introduce provenance-preserving verification cache v3. Store CachedBackendExecution instead of result-only payloads so cache hits replay original attempts and tool provenance rather than re-inferring native execution from the current environment. --- ovk/core/result_cache.py | 34 +++++++-- tests/test_verification_cache.py | 122 +++++++++++++++++++++++++++++-- 2 files changed, 143 insertions(+), 13 deletions(-) diff --git a/ovk/core/result_cache.py b/ovk/core/result_cache.py index 9cadd5d..85f5b3d 100644 --- a/ovk/core/result_cache.py +++ b/ovk/core/result_cache.py @@ -24,12 +24,15 @@ from ovk.core.execution_models import ( BackendEnvironmentFingerprint, BackendObligation, + CachedBackendExecution, + ExecutionAttempt, NormalizedBackendResult, RoutingDecision, VerificationObligation, ) -CACHE_SCHEMA_VERSION = "ovk.cache.v2" +CACHE_SCHEMA_VERSION = "ovk.cache.v3" +CACHE_SCHEMA_VERSION_V2 = "ovk.cache.v2" DEFAULT_CACHE_DIR = Path(".verification/cache") DEFAULT_TTL_SECONDS = 86400 @@ -262,18 +265,37 @@ def get(self, components: dict[str, Any]) -> CacheEntry | None: def put_backend_result( self, components: dict[str, Any], - result: NormalizedBackendResult, + result: CachedBackendExecution, *, meta: dict[str, Any] | None = None, ) -> str: + if not isinstance(result, CachedBackendExecution): + raise TypeError("ovk.cache.v3 requires CachedBackendExecution; v2 result-only entries are invalid") return self.put(components, result.model_dump(mode="json"), meta=meta) def get_backend_result(self, components: dict[str, Any]) -> NormalizedBackendResult | None: + cached = self.get_cached_execution(components) + return None if cached is None else cached.normalized_result + + def get_cached_execution(self, components: dict[str, Any]) -> CachedBackendExecution | None: + """Return a provenance-preserving v3 cache hit, or None. + + Explicitly invalidates v2 (result-only) entries. + """ entry = self.get(components) if entry is None: return None + payload = entry.payload + schema = payload.get("schema_version") + if schema == CACHE_SCHEMA_VERSION_V2 or schema != CACHE_SCHEMA_VERSION: + # Invalidate non-v3 entries so provenance cannot be reconstructed incorrectly. + namespace = str(components.get("namespace") or NAMESPACE_BACKEND_RESULTS) + key_digest = digest_key_components(components) + path = self.namespace_dir(namespace) / f"{key_digest}.json" + path.unlink(missing_ok=True) + return None try: - return NormalizedBackendResult.model_validate(entry.payload) + return CachedBackendExecution.model_validate(payload) except Exception: # noqa: BLE001 - corrupt cache is a miss return None @@ -366,13 +388,13 @@ def __init__(self, hardened: HardenedResultCache | None = None) -> None: def bind_components(self, key: str, components: dict[str, Any]) -> None: self._last_components[key] = components - def get(self, key: str) -> NormalizedBackendResult | None: + def get(self, key: str) -> CachedBackendExecution | None: components = self._last_components.get(key) if components is None: return None - return self._cache.get_backend_result(components) + return self._cache.get_cached_execution(components) - def put(self, key: str, value: NormalizedBackendResult, *, meta: dict[str, Any]) -> None: + def put(self, key: str, value: CachedBackendExecution, *, meta: dict[str, Any]) -> None: components = self._last_components.get(key) if components is None: return diff --git a/tests/test_verification_cache.py b/tests/test_verification_cache.py index 7776fb3..58dab0b 100644 --- a/tests/test_verification_cache.py +++ b/tests/test_verification_cache.py @@ -12,6 +12,8 @@ BackendEnvironmentFingerprint, BackendObligation, BackendSelection, + CachedBackendExecution, + ExecutionAttempt, ExecutionBudget, FallbackPolicy, NormalizedBackendResult, @@ -24,6 +26,7 @@ from ovk.core.models import RiskSeverity, VerificationStatus, VerificationSubject from ovk.core.result_cache import ( CACHE_SCHEMA_VERSION, + CACHE_SCHEMA_VERSION_V2, NAMESPACE_BACKEND_RESULTS, HardenedResultCache, build_aggregate_key_components, @@ -118,6 +121,35 @@ def _fingerprint() -> BackendEnvironmentFingerprint: ) +def _cached_execution(result: NormalizedBackendResult, *, native: bool = False) -> CachedBackendExecution: + attempt = ExecutionAttempt( + attempt_id=result.attempt_id, + backend_obligation_id="bo-1", + backend=result.backend, + required=True, + started_at="2026-01-01T00:00:00Z", + finished_at="2026-01-01T00:00:01Z", + duration_ms=10.0, + termination="completed", + native_execution=native, + tool_version="0.1.0", + tool_digest="tool-digest", + exit_code=0, + raw_result_digest="raw-digest", + ) + return CachedBackendExecution( + attempt=attempt, + native_execution=native, + tool_version=attempt.tool_version, + tool_digest=attempt.tool_digest, + termination=attempt.termination, + exit_code=attempt.exit_code, + raw_result_digest=attempt.raw_result_digest, + environment_fingerprint="env-1", + normalized_result=result, + ) + + def test_key_components_include_required_fields() -> None: obligation = _obligation() routing = _routing(obligation) @@ -166,7 +198,7 @@ def test_subject_mismatch_is_cache_miss(tmp_path: Path) -> None: status=VerificationStatus.PASS, guarantee_type="exposure_graph_check", ) - cache.put_backend_result(components, result) + cache.put_backend_result(components, _cached_execution(result)) other = _obligation(repo="other/repo") other_components = build_backend_result_key_components( @@ -207,11 +239,13 @@ def test_routing_mismatch_is_cache_miss(tmp_path: Path) -> None: ) cache.put_backend_result( components, - NormalizedBackendResult( - attempt_id="a1", - backend="infrastructure-deterministic", - status=VerificationStatus.PASS, - guarantee_type="exposure_graph_check", + _cached_execution( + NormalizedBackendResult( + attempt_id="a1", + backend="infrastructure-deterministic", + status=VerificationStatus.PASS, + guarantee_type="exposure_graph_check", + ) ), ) mismatched = dict(components) @@ -298,11 +332,85 @@ def test_round_trip_backend_result(tmp_path: Path) -> None: guarantee_type="exposure_graph_check", counterexamples=[{"summary": "exposed"}], ) - cache.put_backend_result(components, result) + cache.put_backend_result(components, _cached_execution(result)) loaded = cache.get_backend_result(components) assert loaded is not None assert loaded.status == VerificationStatus.FAIL assert loaded.counterexamples == [{"summary": "exposed"}] + cached = cache.get_cached_execution(components) + assert cached is not None + assert cached.native_execution is False + assert cached.schema_version == CACHE_SCHEMA_VERSION + + +def test_v2_cache_entries_are_invalidated(tmp_path: Path) -> None: + cache = HardenedResultCache(tmp_path) + obligation = _obligation() + routing = _routing(obligation) + backend_obligation = _backend_obligation(obligation, routing) + components = build_backend_result_key_components( + obligation=obligation, + routing=routing, + backend_obligation=backend_obligation, + fingerprint=_fingerprint(), + ) + key = digest_key_components(components) + path = cache.namespace_dir(NAMESPACE_BACKEND_RESULTS) / f"{key}.json" + import json + + path.write_text( + json.dumps( + { + "cached_at": 1.0, + "key_digest": key, + "key_components": components, + "payload": { + "schema_version": CACHE_SCHEMA_VERSION_V2, + "normalized_result": { + "attempt_id": "a1", + "backend": "infrastructure-deterministic", + "status": "pass", + "guarantee_type": "exposure_graph_check", + "assumptions": [], + "limits": [], + "counterexamples": [], + "generated_artifacts": [], + }, + }, + "meta": {}, + } + ), + encoding="utf-8", + ) + assert cache.get_cached_execution(components) is None + assert not path.exists() + + +def test_cache_hit_preserves_native_provenance(tmp_path: Path) -> None: + """Adversarial: current tool availability must not rewrite cached native flag.""" + cache = HardenedResultCache(tmp_path) + obligation = _obligation() + routing = _routing(obligation) + backend_obligation = _backend_obligation(obligation, routing) + components = build_backend_result_key_components( + obligation=obligation, + routing=routing, + backend_obligation=backend_obligation, + fingerprint=_fingerprint(), + ) + result = NormalizedBackendResult( + attempt_id="native-a1", + backend="infrastructure-deterministic", + status=VerificationStatus.PASS, + guarantee_type="exposure_graph_check", + ) + cache.put_backend_result(components, _cached_execution(result, native=True)) + # Simulate environment where native is no longer available — hit must replay True. + loaded = cache.get_cached_execution(components) + assert loaded is not None + assert loaded.native_execution is True + assert loaded.attempt.native_execution is True + assert loaded.tool_digest == "tool-digest" def test_unknown_namespace_rejected(tmp_path: Path) -> None: From 1cfc87bab475df4c19cd10bbbeaf6324ae95923f Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:19 -0700 Subject: [PATCH 09/26] Enforce coverage requirements in typed routing. Reject unsupported coverage as required primaries and stop hardcoding coverage_requirements_met so router assessments reflect real material and backend support. --- ovk/core/router.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/ovk/core/router.py b/ovk/core/router.py index 2b259e6..7ffa183 100644 --- a/ovk/core/router.py +++ b/ovk/core/router.py @@ -237,6 +237,32 @@ def select_primary_with_optional_corroboration( ) ) continue + if not assessment.coverage_requirements_met: + score = _adjust_score_for_preferences( + backend=assessment.backend, + score=float(assessment.score), + prefer_deterministic=config.prefer_deterministic, + ) + if config.accept_partial_primary: + eligible.append( + BackendCandidate( + backend=assessment.backend, + score=score, + support="partial", + guarantee_type=assessment.guarantee_type, + reasons=reasons + ["incomplete coverage; not eligible as required primary"], + native_available=assessment.native_available, + ) + ) + else: + rejected.append( + BackendRejection( + backend=assessment.backend, + reason="coverage requirements not met", + support=assessment.support, + ) + ) + continue if not _guarantee_acceptable(assessment.guarantee_type, acceptable_guarantees): rejected.append( BackendRejection( @@ -289,7 +315,12 @@ def select_primary_with_optional_corroboration( eligible_sorted = sorted(eligible, key=lambda item: (-item.score, item.backend)) primary_candidates = [item for item in eligible_sorted if item.support == "supported"] if not primary_candidates and config.accept_partial_primary: - primary_candidates = list(eligible_sorted) + primary_candidates = [ + item + for item in eligible_sorted + if item.support == "partial" + and not any("incomplete coverage" in reason for reason in item.reasons) + ] if primary_candidates: primary = primary_candidates[0] @@ -482,13 +513,16 @@ def _manifest_assessment( prefer_deterministic=budget.prefer_deterministic, ) + material_requirements_met = domain_match and kind_match + coverage_requirements_met = support == "supported" and material_requirements_met + return BackendCapabilityAssessment( backend=tool_name, support=support, score=score, guarantee_type=guarantee, - material_requirements_met=True, - coverage_requirements_met=True, + material_requirements_met=material_requirements_met, + coverage_requirements_met=coverage_requirements_met, native_available=False, estimated_wall_time_seconds=cost * 30, estimated_memory_mb=256, From 9f537445092a151974e2396cf2d185e9ca5948ab Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:19 -0700 Subject: [PATCH 10/26] Apply cause-specific fallback policy in aggregation. Block timeouts, tool errors, invalid output, and resource exhaustion from becoming accepted passing fallbacks under FallbackPolicy v2. --- ovk/core/backend_aggregation.py | 115 ++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 4 deletions(-) diff --git a/ovk/core/backend_aggregation.py b/ovk/core/backend_aggregation.py index 9a67885..fa3c894 100644 --- a/ovk/core/backend_aggregation.py +++ b/ovk/core/backend_aggregation.py @@ -8,11 +8,26 @@ from dataclasses import dataclass from typing import Any, Sequence -from ovk.core.execution_models import BackendSelection, NormalizedBackendResult +from ovk.core.execution_models import ( + BackendSelection, + ExecutionAttempt, + FallbackPolicy, + NormalizedBackendResult, + TerminationKind, +) from ovk.core.models import MergeRecommendation, VerificationStatus AGGREGATION_FAIL_DOMINANT_V1 = "ovk.aggregate.fail_dominant.v1" +FALLBACK_BLOCKING_TERMINATIONS: frozenset[TerminationKind] = frozenset( + { + "timeout", + "tool_error", + "invalid_output", + "resource_exhausted", + } +) + @dataclass(frozen=True) class AggregationOutcome: @@ -24,6 +39,60 @@ class AggregationOutcome: disagreement: dict[str, Any] | None = None warnings: tuple[str, ...] = () quality_error: bool = False + fallback_used: bool = False + fallback_accepted: bool = False + fallback_cause: str | None = None + + +def evaluate_fallback_acceptance( + *, + policy: FallbackPolicy, + selected: Sequence[BackendSelection], + attempts: Sequence[ExecutionAttempt], + results: Sequence[NormalizedBackendResult], + acceptable_guarantees: Sequence[str] | None = None, +) -> tuple[bool, bool, str | None]: + """Decide whether weaker fallback evidence may satisfy guarantee requirements (INV-017).""" + acceptable = set(acceptable_guarantees or []) + if not acceptable: + return False, False, None + + attempts_by_backend = {item.backend: item for item in attempts} + fallback_used = False + fallback_cause: str | None = None + + for selection in selected: + if not selection.required: + continue + result = next((item for item in results if item.backend == selection.backend), None) + attempt = attempts_by_backend.get(selection.backend) + if result is None or attempt is None: + continue + if result.status != VerificationStatus.PASS: + continue + if result.guarantee_type in acceptable: + continue + + fallback_used = True + fallback_cause = attempt.termination + + if attempt.termination in FALLBACK_BLOCKING_TERMINATIONS: + return True, False, attempt.termination + if not policy.allow_fallback: + return True, False, attempt.termination + if policy.outcome_for_termination(attempt.termination) in {"fail", "error"}: + return True, False, attempt.termination + if policy.fallback_backends and selection.backend not in policy.fallback_backends: + return True, False, attempt.termination + if ( + policy.acceptable_fallback_guarantees + and result.guarantee_type not in policy.acceptable_fallback_guarantees + ): + return True, False, attempt.termination + + if fallback_used: + return True, True, fallback_cause + return False, False, None def build_disagreement_artifact( @@ -58,7 +127,9 @@ def aggregate_fail_dominant_v1( selected: Sequence[BackendSelection], results: Sequence[NormalizedBackendResult], acceptable_guarantees: Sequence[str] | None = None, - fallback_accepted: bool = False, + fallback_accepted: bool | None = None, + fallback_policy: FallbackPolicy | None = None, + attempts: Sequence[ExecutionAttempt] | None = None, ) -> AggregationOutcome: """Apply the fail-dominant aggregation decision table. @@ -75,6 +146,20 @@ def aggregate_fail_dominant_v1( * optional unknown/error warns without invalidating required pass * optional pass cannot upgrade required unknown """ + policy = fallback_policy or FallbackPolicy() + if attempts is not None: + fallback_used, resolved_fallback_accepted, fallback_cause = evaluate_fallback_acceptance( + policy=policy, + selected=selected, + attempts=attempts, + results=results, + acceptable_guarantees=acceptable_guarantees, + ) + else: + fallback_used = False + fallback_cause = None + resolved_fallback_accepted = bool(fallback_accepted) + selected_required = [item for item in selected if item.required] selected_optional = [item for item in selected if not item.required] by_backend = _statuses_by_backend(results) @@ -121,6 +206,9 @@ def aggregate_fail_dominant_v1( merge_recommendation=MergeRecommendation.BLOCK, reason="optional corroborator reported fail", disagreement=disagreement, + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) if any(item.status == VerificationStatus.FAIL for item in required_results): @@ -135,6 +223,9 @@ def aggregate_fail_dominant_v1( merge_recommendation=MergeRecommendation.BLOCK, reason="required backend reported fail", disagreement=disagreement, + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) non_pass = { @@ -153,6 +244,9 @@ def aggregate_fail_dominant_v1( merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, reason="required backend reported unknown, error, or skipped", warnings=tuple(warnings), + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) if not required_results and not selected_required: @@ -161,12 +255,15 @@ def aggregate_fail_dominant_v1( status=VerificationStatus.UNKNOWN, merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, reason="no required backends were selected", + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) # Check guarantees / fallback acceptance for required passes. acceptable = set(acceptable_guarantees or []) for item in required_results: - if acceptable and item.guarantee_type not in acceptable and not fallback_accepted: + if acceptable and item.guarantee_type not in acceptable and not resolved_fallback_accepted: return AggregationOutcome( status=VerificationStatus.UNKNOWN, merge_recommendation=MergeRecommendation.REQUIRE_STRONGER_CHECK, @@ -174,6 +271,9 @@ def aggregate_fail_dominant_v1( f"required result from {item.backend} uses guarantee " f"{item.guarantee_type!r} outside acceptable set" ), + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) for item in optional_results: @@ -185,6 +285,9 @@ def aggregate_fail_dominant_v1( merge_recommendation=MergeRecommendation.ALLOW, reason="every required backend passed with acceptable guarantees", warnings=tuple(warnings), + fallback_used=fallback_used, + fallback_accepted=resolved_fallback_accepted, + fallback_cause=fallback_cause, ) @@ -195,7 +298,9 @@ def aggregate_results( results: Sequence[NormalizedBackendResult], policy: str = AGGREGATION_FAIL_DOMINANT_V1, acceptable_guarantees: Sequence[str] | None = None, - fallback_accepted: bool = False, + fallback_accepted: bool | None = None, + fallback_policy: FallbackPolicy | None = None, + attempts: Sequence[ExecutionAttempt] | None = None, ) -> AggregationOutcome: """Dispatch to a versioned aggregation policy.""" if policy != AGGREGATION_FAIL_DOMINANT_V1: @@ -206,4 +311,6 @@ def aggregate_results( results=results, acceptable_guarantees=acceptable_guarantees, fallback_accepted=fallback_accepted, + fallback_policy=fallback_policy, + attempts=attempts, ) From 517b8ab76cd9418d1265a41f7efa9abb43595d17 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:19 -0700 Subject: [PATCH 11/26] Wire control plane cache replay and compiler contracts. Replay stored cache attempts with intact provenance, surface fallback policy outcomes, and skip execution on compiler vs routing guarantee mismatches instead of silent rewrite. --- ovk/core/backend_control_plane.py | 111 +++++++++++++++++++++++------- 1 file changed, 87 insertions(+), 24 deletions(-) diff --git a/ovk/core/backend_control_plane.py b/ovk/core/backend_control_plane.py index a3a5fea..12ca59b 100644 --- a/ovk/core/backend_control_plane.py +++ b/ovk/core/backend_control_plane.py @@ -24,6 +24,7 @@ from ovk.core.execution_models import ( BackendEnvironmentFingerprint, BackendObligation, + CachedBackendExecution, ExecutionAttempt, ExecutionBudget, NormalizedBackendResult, @@ -39,11 +40,11 @@ class ResultCache(Protocol): - """Minimal cache protocol used by the control plane.""" + """Minimal cache protocol used by the control plane (ovk.cache.v3).""" - def get(self, key: str) -> NormalizedBackendResult | None: ... + def get(self, key: str) -> CachedBackendExecution | None: ... - def put(self, key: str, value: NormalizedBackendResult, *, meta: dict[str, Any]) -> None: ... + def put(self, key: str, value: CachedBackendExecution, *, meta: dict[str, Any]) -> None: ... _CACHE_UNSET = object() @@ -136,6 +137,37 @@ def _error_raw( return raw.model_copy(update=compute_raw_execution_digests(raw)) +def _compiler_contract_error_raw( + *, + backend: str, + backend_obligation_id: str, + message: str, + started_at: str, + started_perf: float, +) -> RawBackendExecution: + finished_at = _utc_now_iso() + raw = RawBackendExecution( + backend=backend, + backend_obligation_id=backend_obligation_id, + termination="invalid_output", + native_execution=False, + exit_code=1, + stderr=message, + raw_result={ + "status": "error", + "error": { + "category": "compiler_contract", + "message": message, + "stage": "compile", + }, + }, + started_at=started_at, + finished_at=finished_at, + duration_ms=(time.perf_counter() - started_perf) * 1000.0, + ) + return raw.model_copy(update=compute_raw_execution_digests(raw)) + + def _attempt_from_raw( *, raw: RawBackendExecution, @@ -235,7 +267,8 @@ def execute( results=results, policy=routing.aggregation_policy, acceptable_guarantees=obligation.acceptable_guarantees, - fallback_accepted=routing.fallback_policy.allow_fallback, + fallback_policy=routing.fallback_policy, + attempts=attempts, ) open_obligations: list[dict[str, Any]] = [] if outcome.disagreement is not None: @@ -261,6 +294,9 @@ def execute( merge_recommendation=outcome.merge_recommendation, aggregation_reason=outcome.reason, open_obligations=open_obligations, + fallback_used=outcome.fallback_used, + fallback_accepted=outcome.fallback_accepted, + fallback_cause=outcome.fallback_cause, ) def _execute_one( @@ -292,8 +328,35 @@ def _execute_one( f"{selection_backend!r}" ) if compiled.expected_guarantee != expected_guarantee and expected_guarantee: - # Record expected guarantee from routing when adapter differs only by alias. - compiled = compiled.model_copy(update={"expected_guarantee": expected_guarantee}) + message = ( + "compiler contract violation: compiled guarantee " + f"{compiled.expected_guarantee!r} does not match routing " + f"expected guarantee {expected_guarantee!r}" + ) + raw = _compiler_contract_error_raw( + backend=selection_backend, + backend_obligation_id=compiled.backend_obligation_id, + message=message, + started_at=started_at, + started_perf=started_perf, + ) + attempt = _attempt_from_raw(raw=raw, required=required) + result = NormalizedBackendResult( + attempt_id=attempt.attempt_id, + backend=selection_backend, + status=VerificationStatus.UNKNOWN, + guarantee_type=expected_guarantee, + assumptions=[], + limits=["compiler guarantee mismatch; execution skipped"], + counterexamples=[ + { + "summary": message, + "failure_mode": "compiler_contract_violation", + } + ], + generated_artifacts=[], + ) + return attempt, result, compiled fingerprint = adapter.fingerprint(compiled) components = control_plane_cache_components( @@ -314,37 +377,37 @@ def _execute_one( bind(key, components) cached = cache.get(key) if cached is not None: - attempt = ExecutionAttempt( - attempt_id="pending", - backend_obligation_id=compiled.backend_obligation_id, - backend=selection_backend, - required=required, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started_perf) * 1000.0, - termination="completed", - native_execution=fingerprint.native_available, - tool_version=fingerprint.tool_version, - tool_digest=fingerprint.tool_digest, - worker_image_digest=fingerprint.worker_image_digest, - raw_result_digest=content_digest({"cache_hit": True, "key": key}), + # Replay stored provenance; never re-infer native_execution. + stored_attempt = cached.attempt + result = cached.normalized_result.model_copy( + update={"attempt_id": stored_attempt.attempt_id} ) - attempt = attempt.model_copy(update={"attempt_id": compute_attempt_id(attempt)}) - result = cached.model_copy(update={"attempt_id": attempt.attempt_id}) - return attempt, result, compiled + return stored_attempt, result, compiled raw = self._run_adapter(adapter, compiled, budget) normalized = adapter.normalize(raw, compiled) attempt = _attempt_from_raw(raw=raw, required=required) result = normalized.model_copy(update={"attempt_id": attempt.attempt_id}) if cache is not None: + cached_exec = CachedBackendExecution( + attempt=attempt, + native_execution=attempt.native_execution, + tool_version=attempt.tool_version, + tool_digest=attempt.tool_digest, + termination=attempt.termination, + exit_code=attempt.exit_code, + raw_result_digest=attempt.raw_result_digest, + environment_fingerprint=fingerprint.environment_digest, + normalized_result=result, + ) cache.put( key, - result, + cached_exec, meta={ "environment_digest": fingerprint.environment_digest, "raw_result_digest": raw.raw_result_digest, "created_at": _utc_now_iso(), + "cache_schema_version": "ovk.cache.v3", }, ) return attempt, result, compiled From 46b20f0097aa9bb4303ef412c58ef4f02dd380e8 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:20 -0700 Subject: [PATCH 12/26] Default self-protection metadata_trusted to false. Require explicit provenance policy before treating branch-protection metadata as trusted so current-state-only signals cannot authorize allow decisions. --- ovk/core/self_protection_compiler.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/ovk/core/self_protection_compiler.py b/ovk/core/self_protection_compiler.py index ac1b480..86dea3e 100644 --- a/ovk/core/self_protection_compiler.py +++ b/ovk/core/self_protection_compiler.py @@ -17,6 +17,29 @@ COMPILER_ID = "ovk.self_protection.neutral.v1" COMPILER_VERSION = "0.1.0" +TRUSTED_METADATA_PROVENANCE_KINDS: frozenset[str] = frozenset( + { + "protected_base_workflow", + "signed_service", + "maintainer_supplied", + } +) + + +def resolve_metadata_trusted(policy: dict[str, Any] | None) -> bool: + """Return True only when policy supplies explicit trusted provenance.""" + if not isinstance(policy, dict): + return False + trust = policy.get("trust") + if not isinstance(trust, dict): + return False + if not bool(trust.get("metadata_trusted")): + return False + provenance = trust.get("provenance_kind") or trust.get("provenance") + if provenance not in TRUSTED_METADATA_PROVENANCE_KINDS: + return False + return True + def _phase(data: dict[str, Any], name: str) -> dict[str, Any]: value = data.get(name, {}) @@ -30,12 +53,13 @@ def compile_self_protection_obligation( head_sha: str, base_sha: str | None = None, policy_digest: str | None = None, - metadata_trusted: bool = True, + metadata_trusted: bool = False, ) -> VerificationObligation: """Compile a self-protection obligation with base/head metadata materials. When ``metadata_trusted`` is True, before/after required-check materials are marked trusted. Untrusted metadata cannot authorize allow under enforcement. + Trust requires explicit policy provenance via ``resolve_metadata_trusted``. """ before = _phase(data, "before") after = _phase(data, "after") From f0ca7ebf701253d7a259d9893c256af31552ef2b Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:20 -0700 Subject: [PATCH 13/26] Add single authoritative routing pipeline. Compile typed obligations before routing and share one routing_id across kernel, planner, and MCP so compat and typed paths cannot diverge. --- ovk/core/kernel.py | 37 +++- ovk/core/planner.py | 24 +++ ovk/core/routing_pipeline.py | 341 +++++++++++++++++++++++++++++++++++ ovk/mcp_server.py | 27 ++- 4 files changed, 418 insertions(+), 11 deletions(-) create mode 100644 ovk/core/routing_pipeline.py diff --git a/ovk/core/kernel.py b/ovk/core/kernel.py index 0d89915..b4201dc 100644 --- a/ovk/core/kernel.py +++ b/ovk/core/kernel.py @@ -22,6 +22,7 @@ from ovk.core.surface_routing import surface_backend_bonuses from ovk.core.result_cache import DEFAULT_CACHE_DIR from ovk.core.risk_ranker import rank_intents +from ovk.core.routing_pipeline import AuthoritativeRoutingPlan, build_authoritative_routing_plan from ovk.core.router import VerificationBudget, route_intent from ovk.paths import resource_path @@ -108,14 +109,6 @@ def execute_kernel( base_sha=base_sha, ) - routing, routing_by_intent = _routing_for_plan( - plan, - context=ctx, - budget=budget, - template_dir=template_dir or resource_path("templates"), - adapter_dir=adapter_dir or resource_path("adapters"), - ) - compiler = ObligationCompilerRegistry.default() obligations = compiler.compile( plan, @@ -127,6 +120,31 @@ def execute_kernel( ) decision_options = bundle_decision_options(ctx.policy) + policy_dict = ctx.policy if isinstance(ctx.policy, dict) else None + + authoritative_plan: AuthoritativeRoutingPlan | None = None + routing_by_intent: dict[str, Any] = {} + routing: list[dict[str, Any]] = [] + + if obligations: + authoritative_plan = build_authoritative_routing_plan( + obligations, + policy=policy_dict, + repo=ctx.repo, + head_sha=ctx.head_sha, + base_sha=ctx.base_sha, + ) + routing_by_intent = authoritative_plan.legacy_routing_by_intent() + routing = authoritative_plan.routing_metadata_list() + else: + routing, routing_by_intent = _routing_for_plan( + plan, + context=ctx, + budget=budget, + template_dir=template_dir or resource_path("templates"), + adapter_dir=adapter_dir or resource_path("adapters"), + ) + if not obligations: bundle = make_bundle( [ @@ -150,7 +168,8 @@ def execute_kernel( cache_dir=cache_dir, use_cache=use_cache, parallel=parallel, - policy=ctx.policy if isinstance(ctx.policy, dict) else None, + policy=policy_dict, + evidence_schema_version="ovk.evidence.v3", ) bundle = make_bundle(evidence_items, **decision_options) diff --git a/ovk/core/planner.py b/ovk/core/planner.py index 7231598..e6d6040 100644 --- a/ovk/core/planner.py +++ b/ovk/core/planner.py @@ -16,9 +16,33 @@ from ovk.core.diff_parser import extract_changed_paths, is_unified_diff from ovk.core.intent_registry import IntentRegistry from ovk.core.router import route_intent +from ovk.core.routing_pipeline import build_authoritative_routing_plan from ovk.paths import resource_path +def plan_from_lane_obligations( + obligations: list[dict[str, Any]], + *, + policy: dict[str, Any] | None = None, + repo: str = "unknown/repo", + head_sha: str = "unknown", + base_sha: str | None = None, +) -> dict[str, Any]: + """Build a plan whose routing uses compile-then-route authoritative decisions.""" + plan = build_authoritative_routing_plan( + obligations, + policy=policy, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + ) + return { + "obligations": obligations, + "routing": plan.routing_metadata_list(), + "routing_by_intent": plan.legacy_routing_by_intent(), + } + + def plan_from_changed_files( changed_files: list[str], *, diff --git a/ovk/core/routing_pipeline.py b/ovk/core/routing_pipeline.py new file mode 100644 index 0000000..7818579 --- /dev/null +++ b/ovk/core/routing_pipeline.py @@ -0,0 +1,341 @@ +"""Single authoritative routing pipeline for typed obligations. + +Compiles backend-neutral obligations before routing, routes each obligation +exactly once via ``route_obligation``, and exposes the same ``routing_id`` to +kernel, CLI, MCP, and evidence emitters. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Mapping + +from ovk.adapters.authorization import build_authorization_registry +from ovk.adapters.ci_secrets import build_ci_secrets_registry +from ovk.adapters.deployment import build_deployment_registry +from ovk.adapters.infrastructure import build_infrastructure_registry +from ovk.adapters.self_protection import build_self_protection_registry +from ovk.core.authorization_compiler import compile_authorization_obligation +from ovk.core.backend_registry import BackendRegistry +from ovk.core.ci_secrets_compiler import compile_ci_secrets_obligation +from ovk.core.deployment_compiler import compile_deployment_obligation +from ovk.core.execution_budget import execution_budget_from_policy +from ovk.core.execution_models import ExecutionContext, RoutingDecision, VerificationObligation +from ovk.core.infrastructure_compiler import compile_infrastructure_obligation +from ovk.core.policy_config import routing_enforced_for_lane +from ovk.core.router import RoutingConfig, route_obligation, routing_config_from_policy, routing_decision_to_legacy_dict +from ovk.core.self_protection_compiler import compile_self_protection_obligation, resolve_metadata_trusted + +LANE_TO_INTENT = { + "self_protection": "agent-cannot-disable-own-ci-gate", + "authorization": "no-admin-route-bypass", + "infrastructure": "no-public-sensitive-resource", + "ci_secrets": "no-secrets-in-untrusted-context", + "deployment": "no-skipped-approval-state", +} + + +def intent_id_for_obligation(obligation: dict[str, Any]) -> str: + """Resolve the canonical intent id for a lane obligation dict.""" + lane = str(obligation["lane"]) + return str(obligation.get("intent_id") or LANE_TO_INTENT.get(lane, lane)) + +RegistryBuilder = Callable[[], BackendRegistry] +CompilerFn = Callable[..., VerificationObligation] + +_LANE_REGISTRY: dict[str, RegistryBuilder] = { + "authorization": build_authorization_registry, + "self_protection": build_self_protection_registry, + "infrastructure": build_infrastructure_registry, + "ci_secrets": build_ci_secrets_registry, + "deployment": build_deployment_registry, +} + + +def _compile_authorization( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, + policy: dict[str, Any] | None, +) -> VerificationObligation: + return compile_authorization_obligation( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + + +def _compile_self_protection( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, + policy: dict[str, Any] | None, +) -> VerificationObligation: + return compile_self_protection_obligation( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + metadata_trusted=resolve_metadata_trusted(policy), + ) + + +def _compile_infrastructure( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, + policy: dict[str, Any] | None, +) -> VerificationObligation: + return compile_infrastructure_obligation( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + + +def _compile_ci_secrets( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, + policy: dict[str, Any] | None, +) -> VerificationObligation: + return compile_ci_secrets_obligation( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + + +def _compile_deployment( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, + policy: dict[str, Any] | None, +) -> VerificationObligation: + return compile_deployment_obligation( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + + +_LANE_COMPILERS: dict[str, CompilerFn] = { + "authorization": _compile_authorization, + "self_protection": _compile_self_protection, + "infrastructure": _compile_infrastructure, + "ci_secrets": _compile_ci_secrets, + "deployment": _compile_deployment, +} + + +@dataclass(frozen=True) +class AuthoritativeRoutingPlan: + """Typed obligations and immutable routing decisions keyed by intent_id.""" + + typed_obligations: dict[str, VerificationObligation] + routing_by_intent: dict[str, RoutingDecision] + + def routing_metadata_list(self) -> list[dict[str, Any]]: + return [ + routing_decision_to_legacy_dict(decision, intent_id=intent_id) + for intent_id, decision in sorted(self.routing_by_intent.items()) + ] + + def legacy_routing_by_intent(self) -> dict[str, dict[str, Any]]: + return { + intent_id: routing_decision_to_legacy_dict(decision, intent_id=intent_id) + for intent_id, decision in self.routing_by_intent.items() + } + + +def compile_typed_obligation( + *, + lane: str, + data: dict[str, Any], + repo: str, + head_sha: str, + base_sha: str | None = None, + policy: dict[str, Any] | None = None, +) -> VerificationObligation: + """Compile one lane input into a backend-neutral obligation.""" + compiler = _LANE_COMPILERS.get(lane) + if compiler is None: + raise ValueError(f"no typed compiler registered for lane {lane!r}") + return compiler( + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + + +def route_compiled_obligation( + obligation: VerificationObligation, + *, + lane: str, + policy: dict[str, Any] | None = None, +) -> RoutingDecision: + """Route a compiled obligation exactly once through ``route_obligation``.""" + registry_builder = _LANE_REGISTRY.get(lane) + if registry_builder is None: + raise ValueError(f"no registry registered for lane {lane!r}") + registry = registry_builder() + routing_config = routing_config_from_policy(policy) + budget = execution_budget_from_policy(policy) + enforced = routing_enforced_for_lane(policy, lane) + context = ExecutionContext( + subject=obligation.subject, + budget=budget, + policy_digest=obligation.policy_digest, + metadata={"enforced": enforced, "lane": lane}, + ) + config = RoutingConfig( + mode="enforced" if enforced else routing_config.mode, + strategy=routing_config.strategy, + aggregation=routing_config.aggregation, + max_selected_backends=routing_config.max_selected_backends, + prefer_deterministic=routing_config.prefer_deterministic, + allow_fallback=routing_config.allow_fallback, + accept_partial_primary=routing_config.accept_partial_primary, + enforced_lanes=frozenset({lane}) if enforced else routing_config.enforced_lanes, + ) + return route_obligation( + obligation, + registry, + context=context, + config=config, + policy=policy, + ) + + +def build_authoritative_routing_plan( + obligations: list[dict[str, Any]], + *, + policy: dict[str, Any] | None = None, + repo: str, + head_sha: str, + base_sha: str | None = None, +) -> AuthoritativeRoutingPlan: + """Compile obligations before routing; route each obligation exactly once. + + Experimental lanes without a typed compiler (for example ``backend``/CBMC) + are skipped here rather than crashing the production control plane. Those + obligations remain available to legacy evaluators when present. + """ + typed: dict[str, VerificationObligation] = {} + routing: dict[str, RoutingDecision] = {} + for item in obligations: + lane = str(item["lane"]) + if lane not in _LANE_COMPILERS or lane not in _LANE_REGISTRY: + # Fail closed for unknown *production* claims elsewhere; skip + # catalog/experimental lanes that lack typed compilers. + continue + intent_id = intent_id_for_obligation(item) + data = dict(item["input"]) + obligation = compile_typed_obligation( + lane=lane, + data=data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + decision = route_compiled_obligation(obligation, lane=lane, policy=policy) + typed[intent_id] = obligation + routing[intent_id] = decision + return AuthoritativeRoutingPlan(typed_obligations=typed, routing_by_intent=routing) + + +def coerce_routing_decision( + routing: RoutingDecision | Mapping[str, Any] | None, + *, + intent_id: str, +) -> RoutingDecision | None: + """Return a typed routing decision when available.""" + if routing is None: + return None + if isinstance(routing, RoutingDecision): + return routing + try: + return RoutingDecision.model_validate(dict(routing)) + except Exception: + routing_id = routing.get("routing_id") + if not routing_id: + return None + return RoutingDecision.model_validate( + { + **routing, + "obligation_id": routing.get("obligation_id") or intent_id, + } + ) + + +def ensure_authoritative_routing( + obligations: list[dict[str, Any]], + routing_by_intent: Mapping[str, RoutingDecision | Mapping[str, Any] | None] | None, + *, + policy: dict[str, Any] | None, + repo: str, + head_sha: str, + base_sha: str | None = None, +) -> AuthoritativeRoutingPlan: + """Merge caller routing with compile-then-route defaults for all obligations.""" + plan = build_authoritative_routing_plan( + obligations, + policy=policy, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + ) + if not routing_by_intent: + return plan + + merged_routing = dict(plan.routing_by_intent) + merged_typed = dict(plan.typed_obligations) + for obligation in obligations: + intent_id = intent_id_for_obligation(obligation) + provided = routing_by_intent.get(intent_id) + coerced = coerce_routing_decision(provided, intent_id=intent_id) + if coerced is not None: + merged_routing[intent_id] = coerced + return AuthoritativeRoutingPlan( + typed_obligations=merged_typed, + routing_by_intent=merged_routing, + ) + + +def require_routing_decision( + routing: RoutingDecision | Mapping[str, Any] | None, + *, + intent_id: str, + lane: str, + policy: dict[str, Any] | None, +) -> RoutingDecision: + """Fail closed when an enforced lane lacks a pre-computed routing decision.""" + decision = coerce_routing_decision(routing, intent_id=intent_id) + if decision is not None: + return decision + if routing_enforced_for_lane(policy, lane): + raise RuntimeError( + f"enforced lane {lane!r} requires authoritative RoutingDecision for intent {intent_id!r}" + ) + raise RuntimeError(f"missing routing decision for intent {intent_id!r}") diff --git a/ovk/mcp_server.py b/ovk/mcp_server.py index 43e454a..43bebe5 100644 --- a/ovk/mcp_server.py +++ b/ovk/mcp_server.py @@ -190,12 +190,34 @@ def select_backends( intent_id: str, *, changed_files: list[str] | None = None, + lane: str | None = None, + lane_input: dict[str, Any] | None = None, + repo: str = "unknown/repo", + head_sha: str = "unknown", + base_sha: str | None = None, + policy: dict[str, Any] | None = None, ) -> dict[str, Any]: - """Select backends for one intent using capability manifests and repo memory.""" + """Select backends for one intent using the authoritative routing pipeline when possible.""" + from ovk.core.lane_compiler import INTENT_TO_LANE + from ovk.core.router import route_intent, routing_decision_to_legacy_dict + from ovk.core.routing_pipeline import compile_typed_obligation, route_compiled_obligation + + resolved_lane = lane or INTENT_TO_LANE.get(intent_id) + if resolved_lane and lane_input is not None: + obligation = compile_typed_obligation( + lane=resolved_lane, + data=lane_input, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + policy=policy, + ) + decision = route_compiled_obligation(obligation, lane=resolved_lane, policy=policy) + return routing_decision_to_legacy_dict(decision, intent_id=intent_id) + from ovk.core.capabilities import CapabilityRegistry from ovk.core.intent_registry import IntentRegistry from ovk.core.repo_memory import router_historical_priors - from ovk.core.router import route_intent from ovk.core.surface_routing import surface_backend_bonuses intents = IntentRegistry.from_directory(resource_path("templates")) @@ -209,6 +231,7 @@ def select_backends( capabilities.all(), historical_priors=router_historical_priors(), surface_bonuses=bonuses, + policy=policy, ) From 1d33db40172bde5f5bb0ba6668a331895445ae4d Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:33 -0700 Subject: [PATCH 14/26] Add isolated deterministic evaluator worker boundary. Move pure evaluators behind a subprocess entrypoint with externally enforced wall-time and output limits so authoritative deterministic adapters cannot run in-process only. --- ovk/core/deterministic_evaluators.py | 224 +++++++++++++++++++++++ ovk/core/worker_runner.py | 258 +++++++++++++++++++++++++++ ovk/workers/__init__.py | 0 ovk/workers/deterministic_entry.py | 25 +++ 4 files changed, 507 insertions(+) create mode 100644 ovk/core/deterministic_evaluators.py create mode 100644 ovk/core/worker_runner.py create mode 100644 ovk/workers/__init__.py create mode 100644 ovk/workers/deterministic_entry.py diff --git a/ovk/core/deterministic_evaluators.py b/ovk/core/deterministic_evaluators.py new file mode 100644 index 0000000..9c19d71 --- /dev/null +++ b/ovk/core/deterministic_evaluators.py @@ -0,0 +1,224 @@ +"""Pure deterministic evaluator functions executed in isolated workers.""" + +from __future__ import annotations + +from typing import Any + +from ovk.adapters.ci_secrets.exposure import find_ci_secrets_counterexamples +from ovk.adapters.deployment.state_machine import find_skipped_approval_paths +from ovk.adapters.infra.exposure import find_exposure_counterexamples +from ovk.adapters.infra.validation import validate_infra_input +from ovk.adapters.opa.self_protection import ( + find_self_protection_unknowns, + find_self_protection_violations, +) +from ovk.adapters.z3.counterexample import counterexamples_from_obligation +from ovk.adapters.z3.executor import run_authorization_obligation_with_z3 +from ovk.adapters.z3.obligation import build_authorization_obligation +from ovk.adapters.z3.validation import validate_authorization_input + +EVALUATOR_IDS = frozenset( + { + "authorization-deterministic", + "self-protection-deterministic", + "infrastructure-deterministic", + "ci-secrets-deterministic", + "deployment-deterministic", + "z3-authorization-native", + } +) + + +def evaluate_deterministic(evaluator_id: str, payload: dict[str, Any]) -> dict[str, Any]: + """Run one registered evaluator and return a JSON-serializable result envelope.""" + if evaluator_id not in EVALUATOR_IDS: + return { + "termination": "tool_error", + "exit_code": 1, + "raw_result": { + "status": "error", + "reason": f"unknown evaluator_id: {evaluator_id}", + }, + } + dispatch = { + "authorization-deterministic": _evaluate_authorization_deterministic, + "self-protection-deterministic": _evaluate_self_protection_deterministic, + "infrastructure-deterministic": _evaluate_infrastructure_deterministic, + "ci-secrets-deterministic": _evaluate_ci_secrets_deterministic, + "deployment-deterministic": _evaluate_deployment_deterministic, + "z3-authorization-native": _evaluate_z3_authorization_native, + } + return dispatch[evaluator_id](payload) + + +def _evaluate_authorization_deterministic(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + issues = validate_authorization_input(data) + if issues: + return { + "termination": "invalid_output", + "exit_code": 1, + "raw_result": { + "status": "unknown", + "reason": "malformed authorization input", + "issues": issues, + "models": [], + "counterexamples": [], + }, + } + auth_obligation = build_authorization_obligation(data) + counterexamples = counterexamples_from_obligation(auth_obligation) + return { + "termination": "completed", + "exit_code": 0, + "raw_result": { + "status": "fail" if counterexamples else "pass", + "reason": ( + "deterministic violation witness found" + if counterexamples + else "no deterministic violation witness found" + ), + "models": counterexamples, + "counterexamples": counterexamples, + }, + } + + +def _evaluate_self_protection_deterministic(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + violations = find_self_protection_violations(data) + unknowns = find_self_protection_unknowns(data) + if violations: + status = "fail" + counterexamples = [item.as_counterexample() for item in violations] + elif unknowns: + status = "unknown" + counterexamples = [item.as_counterexample() for item in unknowns] + else: + status = "pass" + counterexamples = [] + return { + "termination": "completed", + "exit_code": 0, + "raw_result": {"status": status, "counterexamples": counterexamples}, + } + + +def _evaluate_infrastructure_deterministic(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + issues = validate_infra_input(data) + if issues: + return { + "termination": "invalid_output", + "exit_code": 1, + "raw_result": { + "status": "unknown", + "counterexamples": [ + { + "summary": issue.message, + "failure_mode": "infrastructure_abstraction_invalid", + "path": issue.path, + } + for issue in issues + ], + }, + } + counterexamples = find_exposure_counterexamples(data) + return { + "termination": "completed", + "exit_code": 0, + "raw_result": { + "status": "fail" if counterexamples else "pass", + "counterexamples": counterexamples, + }, + } + + +def _evaluate_ci_secrets_deterministic(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + workflows = data.get("workflows") + if not isinstance(workflows, list) or not workflows: + return { + "termination": "invalid_output", + "exit_code": 1, + "raw_result": { + "status": "unknown", + "counterexamples": [ + { + "summary": "Workflow abstraction is missing or empty.", + "failure_mode": "missing_workflow_abstraction", + } + ], + }, + } + counterexamples = find_ci_secrets_counterexamples(data) + return { + "termination": "completed", + "exit_code": 0, + "raw_result": { + "status": "fail" if counterexamples else "pass", + "counterexamples": counterexamples, + }, + } + + +def _evaluate_deployment_deterministic(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + if not data.get("states") or not data.get("transitions"): + return { + "termination": "invalid_output", + "exit_code": 1, + "raw_result": { + "status": "unknown", + "counterexamples": [ + { + "summary": "State machine abstraction is missing states or transitions.", + "failure_mode": "missing_state_machine_abstraction", + } + ], + }, + } + counterexamples = find_skipped_approval_paths(data) + return { + "termination": "completed", + "exit_code": 0, + "raw_result": { + "status": "fail" if counterexamples else "pass", + "counterexamples": counterexamples, + }, + } + + +def _evaluate_z3_authorization_native(payload: dict[str, Any]) -> dict[str, Any]: + data = dict(payload.get("input") or {}) + issues = validate_authorization_input(data) + if issues: + return { + "termination": "invalid_output", + "exit_code": 1, + "native_execution": False, + "raw_result": { + "status": "unknown", + "reason": "malformed input", + "issues": issues, + "models": [], + }, + } + auth_obligation = build_authorization_obligation(data) + native_raw = run_authorization_obligation_with_z3(auth_obligation) + from ovk.adapters.z3.result import normalize_z3_authorization_result + + normalized = normalize_z3_authorization_result(native_raw) + native_execution = native_raw.get("reason") != "z3-solver is not installed" + termination = "tool_unavailable" if native_raw.get("reason") == "z3-solver is not installed" else "completed" + return { + "termination": termination, + "exit_code": 0 if termination == "completed" else 1, + "native_execution": native_execution, + "raw_result": { + "status": normalized["status"], + "reason": native_raw.get("reason"), + "models": native_raw.get("models", []), + "counterexamples": normalized.get("counterexamples", []), + }, + } diff --git a/ovk/core/worker_runner.py b/ovk/core/worker_runner.py new file mode 100644 index 0000000..84360c1 --- /dev/null +++ b/ovk/core/worker_runner.py @@ -0,0 +1,258 @@ +"""Run backend evaluators behind an externally enforced worker boundary.""" + +from __future__ import annotations + +import json +import sys +import tempfile +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ovk.core.execution_budget import BackendWorker, WorkerResult +from ovk.core.execution_models import RawBackendExecution, compute_raw_execution_digests + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +@dataclass(frozen=True) +class EvaluatorWorkerOutcome: + """Parsed evaluator output plus worker enforcement metadata.""" + + termination: str + exit_code: int + raw_result: dict[str, Any] + native_execution: bool + timed_out: bool + worker_rejected: bool + stderr: str | None + + +def run_evaluator_in_worker( + worker: BackendWorker, + *, + evaluator_id: str, + payload: dict[str, Any], + timeout_seconds: float, + cwd: Path | None = None, +) -> EvaluatorWorkerOutcome: + """Serialize payload, execute evaluator in a subprocess, and parse JSON output.""" + if timeout_seconds <= 0: + return EvaluatorWorkerOutcome( + termination="timeout", + exit_code=1, + raw_result={"status": "unknown", "reason": "budget timeout"}, + native_execution=False, + timed_out=False, + worker_rejected=True, + stderr=f"non-positive wall-time budget rejected: {timeout_seconds}", + ) + + work_cwd = (cwd or Path.cwd()).resolve() + with tempfile.TemporaryDirectory(prefix="ovk-worker-", dir=str(work_cwd)) as tmp: + payload_path = Path(tmp) / "payload.json" + payload_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + command = ( + sys.executable, + "-m", + "ovk.workers.deterministic_entry", + "--evaluator-id", + evaluator_id, + "--payload-file", + str(payload_path), + ) + result = worker.run( + command, + cwd=work_cwd, + timeout_seconds=float(timeout_seconds), + max_stdout_bytes=2_000_000, + max_stderr_bytes=500_000, + ) + return _parse_worker_result(result, evaluator_id=evaluator_id) + + +def _parse_worker_result(result: WorkerResult, *, evaluator_id: str) -> EvaluatorWorkerOutcome: + if result.timed_out: + return EvaluatorWorkerOutcome( + termination="timeout", + exit_code=1, + raw_result={"status": "unknown", "reason": "worker execution timed out"}, + native_execution=False, + timed_out=True, + worker_rejected=False, + stderr=result.stderr or None, + ) + if result.exit_code is None: + return EvaluatorWorkerOutcome( + termination="tool_error", + exit_code=1, + raw_result={ + "status": "error", + "reason": result.stderr.strip() or "worker rejected execution", + }, + native_execution=False, + timed_out=False, + worker_rejected=True, + stderr=result.stderr or None, + ) + if not result.stdout.strip(): + return EvaluatorWorkerOutcome( + termination="invalid_output", + exit_code=1, + raw_result={ + "status": "error", + "reason": result.stderr.strip() or f"{evaluator_id} returned empty output", + }, + native_execution=False, + timed_out=False, + worker_rejected=False, + stderr=result.stderr or None, + ) + try: + envelope = json.loads(result.stdout) + except json.JSONDecodeError: + return EvaluatorWorkerOutcome( + termination="invalid_output", + exit_code=1, + raw_result={ + "status": "error", + "reason": "worker returned invalid JSON", + }, + native_execution=False, + timed_out=False, + worker_rejected=False, + stderr=result.stderr or None, + ) + if not isinstance(envelope, dict): + return EvaluatorWorkerOutcome( + termination="invalid_output", + exit_code=1, + raw_result={"status": "error", "reason": "worker returned non-object JSON"}, + native_execution=False, + timed_out=False, + worker_rejected=False, + stderr=result.stderr or None, + ) + return EvaluatorWorkerOutcome( + termination=str(envelope.get("termination", "completed")), + exit_code=int(envelope.get("exit_code", result.exit_code or 1)), + raw_result=dict(envelope.get("raw_result") or {}), + native_execution=bool(envelope.get("native_execution", False)), + timed_out=False, + worker_rejected=False, + stderr=result.stderr or None, + ) + + +def raw_execution_from_worker_outcome( + *, + backend: str, + backend_obligation_id: str, + adapter_version: str, + outcome: EvaluatorWorkerOutcome, + started_at: str, + duration_ms: float, +) -> RawBackendExecution: + """Project a worker outcome into ``RawBackendExecution``.""" + raw = RawBackendExecution( + backend=backend, + backend_obligation_id=backend_obligation_id, + termination=outcome.termination, # type: ignore[arg-type] + native_execution=outcome.native_execution, + exit_code=outcome.exit_code, + stderr=outcome.stderr, + raw_result=outcome.raw_result, + started_at=started_at, + finished_at=_utc_now_iso(), + duration_ms=duration_ms, + tool_version=adapter_version, + ) + return raw.model_copy(update=compute_raw_execution_digests(raw)) + + +def require_worker( + worker: BackendWorker | None, + *, + backend: str, + backend_obligation_id: str, + adapter_version: str, +) -> BackendWorker: + """Fail closed when an authoritative adapter is invoked without a worker.""" + if worker is not None: + return worker + started_at = _utc_now_iso() + raise WorkerRequiredError( + raw_execution_from_worker_outcome( + backend=backend, + backend_obligation_id=backend_obligation_id, + adapter_version=adapter_version, + outcome=EvaluatorWorkerOutcome( + termination="tool_error", + exit_code=1, + raw_result={ + "status": "error", + "reason": "authoritative adapter requires BackendWorker; in-process execution forbidden", + }, + native_execution=False, + timed_out=False, + worker_rejected=True, + stderr="missing worker", + ), + started_at=started_at, + duration_ms=0.0, + ) + ) + + +class WorkerRequiredError(Exception): + """Raised when an adapter is invoked without a required worker.""" + + def __init__(self, raw: RawBackendExecution) -> None: + super().__init__("authoritative adapter requires BackendWorker") + self.raw = raw + + +def run_with_required_worker( + worker: BackendWorker | None, + *, + backend: str, + backend_obligation_id: str, + adapter_version: str, + evaluator_id: str, + payload: dict[str, Any], + timeout_seconds: float, + cwd: Path | None = None, +) -> RawBackendExecution: + """Execute an evaluator in a worker or return a fail-closed raw execution.""" + started = time.perf_counter() + started_at = _utc_now_iso() + if worker is None: + try: + require_worker( + None, + backend=backend, + backend_obligation_id=backend_obligation_id, + adapter_version=adapter_version, + ) + except WorkerRequiredError as exc: + return exc.raw + assert worker is not None + outcome = run_evaluator_in_worker( + worker, + evaluator_id=evaluator_id, + payload=payload, + timeout_seconds=timeout_seconds, + cwd=cwd, + ) + return raw_execution_from_worker_outcome( + backend=backend, + backend_obligation_id=backend_obligation_id, + adapter_version=adapter_version, + outcome=outcome, + started_at=started_at, + duration_ms=(time.perf_counter() - started) * 1000.0, + ) diff --git a/ovk/workers/__init__.py b/ovk/workers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ovk/workers/deterministic_entry.py b/ovk/workers/deterministic_entry.py new file mode 100644 index 0000000..7ed8b53 --- /dev/null +++ b/ovk/workers/deterministic_entry.py @@ -0,0 +1,25 @@ +"""Subprocess worker entry points for isolated backend evaluation.""" + +from __future__ import annotations + +import argparse +import json +import sys + +from ovk.core.deterministic_evaluators import evaluate_deterministic + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Run an OVK evaluator in an isolated worker process.") + parser.add_argument("--evaluator-id", required=True) + parser.add_argument("--payload-file", required=True) + args = parser.parse_args(argv) + + payload = json.loads(open(args.payload_file, encoding="utf-8").read()) + result = evaluate_deterministic(args.evaluator_id, payload) + sys.stdout.write(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From be6fa8109b098b58a4de0a81ea62ea3373ec5fc1 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:33 -0700 Subject: [PATCH 15/26] Route deterministic lane adapters through worker runners. Require the control-plane worker for authoritative deterministic runs so wall-time, env, and cancellation are enforced outside adapter process logic. --- .../authorization/deterministic_adapter.py | 59 ++++-------------- .../ci_secrets/deterministic_adapter.py | 58 +++++------------- .../deployment/deterministic_adapter.py | 57 +++++------------- .../infrastructure/deterministic_adapter.py | 60 +++++-------------- .../self_protection/deterministic_adapter.py | 55 +++++------------ 5 files changed, 70 insertions(+), 219 deletions(-) diff --git a/ovk/adapters/authorization/deterministic_adapter.py b/ovk/adapters/authorization/deterministic_adapter.py index 39b7508..f2f95b1 100644 --- a/ovk/adapters/authorization/deterministic_adapter.py +++ b/ovk/adapters/authorization/deterministic_adapter.py @@ -6,9 +6,6 @@ from datetime import datetime, timezone from typing import Any -from ovk.adapters.z3.counterexample import counterexamples_from_obligation -from ovk.adapters.z3.obligation import build_authorization_obligation -from ovk.adapters.z3.validation import validate_authorization_input from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -26,9 +23,10 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -161,55 +159,18 @@ def run( self, backend_obligation: BackendObligation, budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, ) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - data = dict(backend_obligation.payload.get("input") or {}) - issues = validate_authorization_input(data) - if issues: - raw_result = { - "status": "unknown", - "reason": "malformed authorization input", - "issues": issues, - "models": [], - } - termination = "invalid_output" - else: - auth_obligation = build_authorization_obligation(data) - counterexamples = counterexamples_from_obligation(auth_obligation) - raw_result = { - "status": "fail" if counterexamples else "pass", - "reason": ( - "deterministic violation witness found" - if counterexamples - else "no deterministic violation witness found" - ), - "models": counterexamples, - "counterexamples": counterexamples, - } - termination = "completed" - # Soft timeout signal for tests: budget of 0 forces timeout. - duration_ms = (time.perf_counter() - started) * 1000.0 - if budget.per_backend_wall_time_seconds <= 0: - termination = "timeout" - raw_result = { - "status": "unknown", - "reason": "budget timeout", - "models": [], - } - raw = RawBackendExecution( + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination=termination, # type: ignore[arg-type] - native_execution=False, - exit_code=0 if termination == "completed" else 1, - raw_result=raw_result, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=duration_ms, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="authorization-deterministic", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, diff --git a/ovk/adapters/ci_secrets/deterministic_adapter.py b/ovk/adapters/ci_secrets/deterministic_adapter.py index 63ed85f..11de591 100644 --- a/ovk/adapters/ci_secrets/deterministic_adapter.py +++ b/ovk/adapters/ci_secrets/deterministic_adapter.py @@ -26,7 +26,9 @@ compute_payload_digest, compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -154,52 +156,22 @@ def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironme native_available=False, ) - def run(self, backend_obligation: BackendObligation, budget: ExecutionBudget) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - if budget.per_backend_wall_time_seconds <= 0: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="timeout", - native_execution=False, - exit_code=1, - raw_result={"status": "unknown", "reason": "budget timeout", "counterexamples": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - data = dict(backend_obligation.payload.get("input") or {}) - workflows = data.get("workflows") - if not isinstance(workflows, list) or not workflows: - status = "unknown" - counterexamples = [ - { - "summary": "Workflow abstraction is missing or empty.", - "failure_mode": "missing_workflow_abstraction", - } - ] - termination = "invalid_output" - else: - counterexamples = find_ci_secrets_counterexamples(data) - status = "fail" if counterexamples else "pass" - termination = "completed" - raw = RawBackendExecution( + def run( + self, + backend_obligation: BackendObligation, + budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, + ) -> RawBackendExecution: + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination=termination, # type: ignore[arg-type] - native_execution=False, - exit_code=0 if termination == "completed" else 1, - raw_result={"status": status, "counterexamples": counterexamples}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="ci-secrets-deterministic", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, diff --git a/ovk/adapters/deployment/deterministic_adapter.py b/ovk/adapters/deployment/deterministic_adapter.py index 8a73a3d..022d7c1 100644 --- a/ovk/adapters/deployment/deterministic_adapter.py +++ b/ovk/adapters/deployment/deterministic_adapter.py @@ -26,7 +26,9 @@ compute_payload_digest, compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -154,51 +156,22 @@ def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironme native_available=False, ) - def run(self, backend_obligation: BackendObligation, budget: ExecutionBudget) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - if budget.per_backend_wall_time_seconds <= 0: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="timeout", - native_execution=False, - exit_code=1, - raw_result={"status": "unknown", "reason": "budget timeout", "counterexamples": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - data = dict(backend_obligation.payload.get("input") or {}) - if not data.get("states") or not data.get("transitions"): - status = "unknown" - counterexamples = [ - { - "summary": "State machine abstraction is missing states or transitions.", - "failure_mode": "missing_state_machine_abstraction", - } - ] - termination = "invalid_output" - else: - counterexamples = find_skipped_approval_paths(data) - status = "fail" if counterexamples else "pass" - termination = "completed" - raw = RawBackendExecution( + def run( + self, + backend_obligation: BackendObligation, + budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, + ) -> RawBackendExecution: + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination=termination, # type: ignore[arg-type] - native_execution=False, - exit_code=0 if termination == "completed" else 1, - raw_result={"status": status, "counterexamples": counterexamples}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="deployment-deterministic", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, diff --git a/ovk/adapters/infrastructure/deterministic_adapter.py b/ovk/adapters/infrastructure/deterministic_adapter.py index 45adc99..fda4c4c 100644 --- a/ovk/adapters/infrastructure/deterministic_adapter.py +++ b/ovk/adapters/infrastructure/deterministic_adapter.py @@ -27,7 +27,9 @@ compute_payload_digest, compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -155,54 +157,22 @@ def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironme native_available=False, ) - def run(self, backend_obligation: BackendObligation, budget: ExecutionBudget) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - if budget.per_backend_wall_time_seconds <= 0: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="timeout", - native_execution=False, - exit_code=1, - raw_result={"status": "unknown", "reason": "budget timeout", "counterexamples": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - data = dict(backend_obligation.payload.get("input") or {}) - issues = validate_infra_input(data) - if issues: - status = "unknown" - counterexamples = [ - { - "summary": issue.message, - "failure_mode": "infrastructure_abstraction_invalid", - "path": issue.path, - } - for issue in issues - ] - termination = "invalid_output" - else: - counterexamples = find_exposure_counterexamples(data) - status = "fail" if counterexamples else "pass" - termination = "completed" - raw = RawBackendExecution( + def run( + self, + backend_obligation: BackendObligation, + budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, + ) -> RawBackendExecution: + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination=termination, # type: ignore[arg-type] - native_execution=False, - exit_code=0 if termination == "completed" else 1, - raw_result={"status": status, "counterexamples": counterexamples}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="infrastructure-deterministic", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, diff --git a/ovk/adapters/self_protection/deterministic_adapter.py b/ovk/adapters/self_protection/deterministic_adapter.py index cf75b0b..27674c0 100644 --- a/ovk/adapters/self_protection/deterministic_adapter.py +++ b/ovk/adapters/self_protection/deterministic_adapter.py @@ -29,7 +29,9 @@ compute_payload_digest, compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -160,49 +162,22 @@ def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironme native_available=False, ) - def run(self, backend_obligation: BackendObligation, budget: ExecutionBudget) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - if budget.per_backend_wall_time_seconds <= 0: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="timeout", - native_execution=False, - exit_code=1, - raw_result={"status": "unknown", "reason": "budget timeout", "counterexamples": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - data = dict(backend_obligation.payload.get("input") or {}) - violations = find_self_protection_violations(data) - unknowns = find_self_protection_unknowns(data) - if violations: - status = "fail" - counterexamples = [item.as_counterexample() for item in violations] - elif unknowns: - status = "unknown" - counterexamples = [item.as_counterexample() for item in unknowns] - else: - status = "pass" - counterexamples = [] - raw = RawBackendExecution( + def run( + self, + backend_obligation: BackendObligation, + budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, + ) -> RawBackendExecution: + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination="completed", - native_execution=False, - exit_code=0, - raw_result={"status": status, "counterexamples": counterexamples}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="self-protection-deterministic", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, From f21b74029badc5d74bf746153639412a011f8015 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:33 -0700 Subject: [PATCH 16/26] Align native and lane adapters with worker isolation. Share the externally enforced worker contract for OPA/Z3 paths and stop post-hoc soft timeout rewriting based only on elapsed wall time after return. --- ovk/adapters/authorization/z3_adapter.py | 82 +++------------------ ovk/adapters/lane/_base.py | 5 -- ovk/adapters/self_protection/opa_adapter.py | 22 +++++- 3 files changed, 31 insertions(+), 78 deletions(-) diff --git a/ovk/adapters/authorization/z3_adapter.py b/ovk/adapters/authorization/z3_adapter.py index 9ad743c..ef6d219 100644 --- a/ovk/adapters/authorization/z3_adapter.py +++ b/ovk/adapters/authorization/z3_adapter.py @@ -29,7 +29,9 @@ compute_payload_digest, compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus +from ovk.core.worker_runner import run_with_required_worker def _utc_now_iso() -> str: @@ -193,82 +195,18 @@ def run( self, backend_obligation: BackendObligation, budget: ExecutionBudget, + *, + worker: BackendWorker | None = None, ) -> RawBackendExecution: - started = time.perf_counter() - started_at = _utc_now_iso() - data = dict(backend_obligation.payload.get("input") or {}) - if budget.per_backend_wall_time_seconds <= 0: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="timeout", - native_execution=True, - exit_code=1, - raw_result={"status": "unknown", "reason": "budget timeout", "models": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - issues = validate_authorization_input(data) - if issues: - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="invalid_output", - native_execution=False, - exit_code=1, - raw_result={"status": "unknown", "reason": "malformed input", "issues": issues, "models": []}, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - if not z3_available(): - # Explicit non-fallback: native adapter reports unavailable/unknown. - raw = RawBackendExecution( - backend=self.backend_id, - backend_obligation_id=backend_obligation.backend_obligation_id, - termination="tool_unavailable", - native_execution=False, - exit_code=1, - raw_result={ - "status": "unknown", - "reason": "z3-solver is not installed", - "models": [], - }, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, - ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) - - auth_obligation = build_authorization_obligation(data) - native_raw = run_authorization_obligation_with_z3(auth_obligation) - normalized = normalize_z3_authorization_result(native_raw) - raw = RawBackendExecution( + return run_with_required_worker( + worker, backend=self.backend_id, backend_obligation_id=backend_obligation.backend_obligation_id, - termination="completed", - native_execution=True, - exit_code=0, - raw_result={ - "status": normalized["status"], - "reason": native_raw.get("reason"), - "models": native_raw.get("models", []), - "counterexamples": normalized.get("counterexamples", []), - }, - started_at=started_at, - finished_at=_utc_now_iso(), - duration_ms=(time.perf_counter() - started) * 1000.0, - tool_version=self.adapter_version, + adapter_version=self.adapter_version, + evaluator_id="z3-authorization-native", + payload=dict(backend_obligation.payload), + timeout_seconds=budget.per_backend_wall_time_seconds, ) - return raw.model_copy(update=compute_raw_execution_digests(raw)) def normalize( self, diff --git a/ovk/adapters/lane/_base.py b/ovk/adapters/lane/_base.py index ebcc2aa..2f08d6b 100644 --- a/ovk/adapters/lane/_base.py +++ b/ovk/adapters/lane/_base.py @@ -249,11 +249,6 @@ def run( finished_at = _utc_now_iso() duration_ms = (time.perf_counter() - started) * 1000.0 - # Soft budget recording only; hard isolation arrives with the control plane. - if duration_ms > budget.per_backend_wall_time_seconds * 1000.0 and termination == "completed": - termination = "timeout" - raw_result["status"] = "unknown" - raw_result["timeout"] = True raw = RawBackendExecution( backend=self.backend_id, diff --git a/ovk/adapters/self_protection/opa_adapter.py b/ovk/adapters/self_protection/opa_adapter.py index 8317633..b7aab3d 100644 --- a/ovk/adapters/self_protection/opa_adapter.py +++ b/ovk/adapters/self_protection/opa_adapter.py @@ -30,6 +30,7 @@ compute_payload_digest, compute_raw_execution_digests, ) +from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus @@ -173,7 +174,8 @@ def run( self, backend_obligation: BackendObligation, budget: ExecutionBudget, - worker=None, + *, + worker: BackendWorker | None = None, ) -> RawBackendExecution: started = time.perf_counter() started_at = _utc_now_iso() @@ -192,6 +194,24 @@ def run( ) return raw.model_copy(update=compute_raw_execution_digests(raw)) + if worker is None: + raw = RawBackendExecution( + backend=self.backend_id, + backend_obligation_id=backend_obligation.backend_obligation_id, + termination="tool_error", + native_execution=False, + exit_code=1, + raw_result={ + "status": "error", + "reason": "authoritative OPA adapter requires BackendWorker", + "counterexamples": [], + }, + started_at=started_at, + finished_at=_utc_now_iso(), + duration_ms=(time.perf_counter() - started) * 1000.0, + ) + return raw.model_copy(update=compute_raw_execution_digests(raw)) + if not opa_available(): raw = RawBackendExecution( backend=self.backend_id, From 4d6fd3348ad718290c0f33c15d46fb300a97dbf8 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:34 -0700 Subject: [PATCH 17/26] Route enforced lanes through authoritative pipeline and trust gates. Use the single routing pipeline for enforced execution and fail closed when self-protection allow decisions lack trusted metadata provenance. --- ovk/core/adapter_runtime.py | 814 +++++++++++++++++++++++------------- 1 file changed, 526 insertions(+), 288 deletions(-) diff --git a/ovk/core/adapter_runtime.py b/ovk/core/adapter_runtime.py index 6aa729f..e7a6747 100644 --- a/ovk/core/adapter_runtime.py +++ b/ovk/core/adapter_runtime.py @@ -1,582 +1,820 @@ """Execute compiled obligations through lane adapters with routing metadata. + + Shadow mode runs the typed control plane beside legacy ``evaluate_lane``; -legacy evidence remains authoritative. + +legacy evidence remains authoritative unless a lane is policy-enforced. + """ + + from __future__ import annotations + + from concurrent.futures import ThreadPoolExecutor + from pathlib import Path -from typing import Any + +from typing import Any, Mapping + + from ovk.adapters.authorization import build_authorization_registry + from ovk.adapters.ci_secrets import build_ci_secrets_registry + from ovk.adapters.deployment import build_deployment_registry + from ovk.adapters.infrastructure import build_infrastructure_registry + from ovk.adapters.lane import build_default_lane_registry + from ovk.adapters.self_protection import build_self_protection_registry -from ovk.core.authorization_compiler import compile_authorization_obligation + from ovk.core.backend_control_plane import BackendControlPlane, compare_shadow_to_legacy + from ovk.core.bundle import content_digest -from ovk.core.ci_secrets_compiler import compile_ci_secrets_obligation -from ovk.core.deployment_compiler import compile_deployment_obligation + from ovk.core.evidence_from_execution import execution_record_to_evidence + from ovk.core.execution_budget import LocalSubprocessWorker, execution_budget_from_policy -from ovk.core.execution_models import ExecutionContext -from ovk.core.infrastructure_compiler import compile_infrastructure_obligation + +from ovk.core.execution_models import ExecutionContext, RoutingDecision, VerificationObligation + from ovk.core.models import VerificationEvidence + from ovk.core.multi_lane import evaluate_lane + from ovk.core.policy_config import resolve_routing_config, routing_enforced_for_lane + from ovk.core.result_cache import ( + ControlPlaneResultCache, + HardenedResultCache, + cache_key, + get_cached_evidence, + store_cached_evidence, + ) -from ovk.core.router import RoutingConfig, route_obligation, routing_config_from_policy -from ovk.core.self_protection_compiler import compile_self_protection_obligation + +from ovk.core.router import RoutingConfig, route_obligation, routing_config_from_policy, routing_decision_to_legacy_dict + +from ovk.core.routing_pipeline import ( + + AuthoritativeRoutingPlan, + + ensure_authoritative_routing, + + intent_id_for_obligation, + + require_routing_decision, + +) + +from ovk.core.self_protection_compiler import resolve_metadata_trusted + from ovk.core.shadow_obligation import build_shadow_obligation + + + def _control_plane(*, cache_dir: Path | None = None) -> BackendControlPlane: + """Build an enforced/shadow control plane with hardened cache + worker.""" + hardened = HardenedResultCache(cache_dir / "control-plane") if cache_dir is not None else HardenedResultCache() + return BackendControlPlane( + cache=ControlPlaneResultCache(hardened), + worker=LocalSubprocessWorker(), + use_hardened_cache=True, + ) + + + + LANE_TO_INTENT = { + "self_protection": "agent-cannot-disable-own-ci-gate", + "authorization": "no-admin-route-bypass", + "infrastructure": "no-public-sensitive-resource", + "ci_secrets": "no-secrets-in-untrusted-context", + "deployment": "no-skipped-approval-state", + +} + + + +_LANE_REGISTRY_BUILDERS = { + + "authorization": build_authorization_registry, + + "self_protection": build_self_protection_registry, + + "infrastructure": build_infrastructure_registry, + + "ci_secrets": build_ci_secrets_registry, + + "deployment": build_deployment_registry, + } + + + +def _routing_metadata( + + routing: RoutingDecision | Mapping[str, Any] | None, + + *, + + intent_id: str, + + routing_enforced: bool, + +) -> dict[str, Any] | None: + + if routing is None: + + return None + + if isinstance(routing, RoutingDecision): + + payload = routing_decision_to_legacy_dict(routing, intent_id=intent_id) + + else: + + payload = dict(routing) + + payload["routing_enforced"] = routing_enforced + + return payload + + + + + def _attach_execution_metadata( + evidence: VerificationEvidence, + *, + lane: str, + data: dict[str, Any], - routing: dict[str, Any] | None, + + routing: RoutingDecision | Mapping[str, Any] | None, + shadow_comparison: dict[str, Any] | None = None, + intent_id: str | None = None, + job_id: str | None = None, + input_format: str | None = None, + + routing_enforced: bool = False, + ) -> VerificationEvidence: + """Record routing, input digest, and obligation-scoped evidence identity.""" - resolved_intent = intent_id or str((routing or {}).get("intent_id") or LANE_TO_INTENT.get(lane, lane)) + + resolved_intent = intent_id or str((routing or {}).get("intent_id") if isinstance(routing, dict) else None) or LANE_TO_INTENT.get(lane, lane) + resolved_format = input_format or "infra" + identity = { + "intent_id": resolved_intent, + "lane": lane, + "input": data, + "input_format": resolved_format, + "job_id": job_id, + } + input_digest = content_digest({"lane": lane, "input": data}) + evidence_suffix = content_digest(identity)[:12] + artifacts = list(evidence.generated_artifacts) + artifacts.append({"kind": "input_digest", "digest": input_digest, "lane": lane}) - if routing is not None: + + metadata = _routing_metadata(routing, intent_id=resolved_intent, routing_enforced=routing_enforced) + + if metadata is not None: + artifacts.append( + { + "kind": "backend_routing", - "intent_id": routing.get("intent_id"), - "selected": routing.get("selected", []), - "rejected": routing.get("rejected", []), - "routing_id": routing.get("routing_id"), - "routing_enforced": bool(routing.get("routing_enforced")), + + "intent_id": metadata.get("intent_id", resolved_intent), + + "selected": metadata.get("selected", []), + + "rejected": metadata.get("rejected", []), + + "routing_id": metadata.get("routing_id") or evidence.routing_id, + + "routing_enforced": routing_enforced, + "executed_backends": [claim.backend for claim in evidence.backend_claims], + } + ) + if shadow_comparison is not None: + artifacts.append(shadow_comparison) + return evidence.model_copy( + update={ + "evidence_id": f"{evidence.evidence_id}-{evidence_suffix}", + "generated_artifacts": artifacts, + + "routing_id": evidence.routing_id or (metadata or {}).get("routing_id"), + } + ) + + + def _legacy_status_and_recommendation(evidence: VerificationEvidence) -> tuple[str, str]: + status = evidence.backend_claims[0].status.value if evidence.backend_claims else "unknown" + recommendation = str(evidence.decision.get("merge_recommendation", "require_human_review")) + return status, recommendation + + + def _run_shadow_path( + *, + lane: str, + data: dict[str, Any], + repo: str, + head_sha: str, + base_sha: str | None, + intent_id: str, + policy: dict[str, Any] | None, + ) -> dict[str, Any] | None: + """Execute the typed control plane for comparison; never raises to legacy.""" + try: + registry = build_default_lane_registry() + obligation = build_shadow_obligation( + lane=lane, + data=data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + intent_id=intent_id, + ) + budget = execution_budget_from_policy(policy) + context = ExecutionContext( + subject=obligation.subject, + budget=budget, + policy_digest=obligation.policy_digest, + metadata={"shadow": True}, + ) + routing = route_obligation(obligation, registry, context=context, policy=policy) + record = _control_plane().execute(obligation, routing, registry=registry) + return { + "record": record, + "routing": routing, + } + except Exception as exc: # noqa: BLE001 - shadow must not affect legacy authority + return { + "error": { + "category": type(exc).__name__, + "message": str(exc), + } + } -def _run_enforced_self_protection( + + + +def _run_enforced_with_routing( + *, + + lane: str, + data: dict[str, Any], - repo: str, - head_sha: str, - base_sha: str | None, + + routing: RoutingDecision, + + typed_obligation: VerificationObligation, + policy: dict[str, Any] | None, + + schema_version: str, + ) -> VerificationEvidence: - """Execute self-protection through registry selection; result is authoritative.""" - registry = build_self_protection_registry() - metadata_trusted = True - if isinstance(policy, dict): - trust = policy.get("trust", {}) - if isinstance(trust, dict) and "metadata_trusted" in trust: - metadata_trusted = bool(trust.get("metadata_trusted")) - routing = policy.get("routing", {}) - if isinstance(routing, dict) and "metadata_trusted" in routing: - metadata_trusted = bool(routing.get("metadata_trusted")) - obligation = compile_self_protection_obligation( - data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - metadata_trusted=metadata_trusted, - ) - # Untrusted metadata cannot produce allow under enforcement. - if not metadata_trusted and obligation.coverage.status != "complete": - pass - budget = execution_budget_from_policy(policy) - routing_config = routing_config_from_policy(policy) - context = ExecutionContext( - subject=obligation.subject, - budget=budget, - policy_digest=obligation.policy_digest, - metadata={"enforced": True, "lane": "self_protection", "metadata_trusted": metadata_trusted}, - ) - routing = route_obligation( - obligation, - registry, - context=context, - config=RoutingConfig( - mode="enforced", - strategy=routing_config.strategy, - aggregation=routing_config.aggregation, - max_selected_backends=routing_config.max_selected_backends, - prefer_deterministic=routing_config.prefer_deterministic, - allow_fallback=routing_config.allow_fallback, - accept_partial_primary=routing_config.accept_partial_primary, - enforced_lanes=frozenset({"self_protection"}), - ), - policy=policy, - ) - record = _control_plane().execute(obligation, routing, registry=registry) + + """Execute one enforced lane using a pre-computed immutable routing decision.""" + + registry_builder = _LANE_REGISTRY_BUILDERS.get(lane) + + if registry_builder is None: + + raise RuntimeError(f"no enforced registry for lane {lane!r}") + + registry = registry_builder() + + record = _control_plane().execute(typed_obligation, routing, registry=registry) + evidence = execution_record_to_evidence( + record, + author_type=str((data.get("actor") or {}).get("type", data.get("author_type", "unknown"))), + agent=str((data.get("actor") or {}).get("id", data.get("agent", "unknown"))), + task=str(data.get("task", "unknown")), + routing_enforced=True, - schema_version="ovk.evidence.v2", + + schema_version=schema_version, + ) - if not metadata_trusted and evidence.decision.get("merge_recommendation") == "allow": - evidence = evidence.model_copy( - update={ - "decision": { - **evidence.decision, - "merge_recommendation": "require_human_review", - "human_review_required": True, - "reason": "untrusted metadata cannot authorize allow under enforcement", - "fallback_accepted": False, + + if lane == "self_protection": + + metadata_trusted = resolve_metadata_trusted(policy) + + if not metadata_trusted and evidence.decision.get("merge_recommendation") == "allow": + + evidence = evidence.model_copy( + + update={ + + "decision": { + + **evidence.decision, + + "merge_recommendation": "require_human_review", + + "human_review_required": True, + + "reason": "untrusted metadata cannot authorize allow under enforcement", + + "fallback_accepted": False, + + } + } - } - ) + + ) + return evidence -def _run_enforced_authorization( + + + +def _evaluate_obligation( + + obligation: dict[str, Any], + *, - data: dict[str, Any], - repo: str, - head_sha: str, - base_sha: str | None, - policy: dict[str, Any] | None, -) -> VerificationEvidence: - """Execute authorization through registry selection; result is authoritative.""" - registry = build_authorization_registry() - obligation = compile_authorization_obligation( - data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - policy=policy, - ) - budget = execution_budget_from_policy(policy) - routing_config = routing_config_from_policy(policy) - context = ExecutionContext( - subject=obligation.subject, - budget=budget, - policy_digest=obligation.policy_digest, - metadata={"enforced": True, "lane": "authorization"}, - ) - routing = route_obligation( - obligation, - registry, - context=context, - config=RoutingConfig( - mode="enforced", - strategy=routing_config.strategy, - aggregation=routing_config.aggregation, - max_selected_backends=routing_config.max_selected_backends, - prefer_deterministic=routing_config.prefer_deterministic, - allow_fallback=routing_config.allow_fallback, - accept_partial_primary=routing_config.accept_partial_primary, - enforced_lanes=frozenset({"authorization"}), - ), - policy=policy, - ) - record = _control_plane().execute(obligation, routing, registry=registry) - return execution_record_to_evidence( - record, - author_type=str(data.get("author_type", "unknown")), - agent=str(data.get("agent", "unknown")), - task=str(data.get("task", "unknown")), - routing_enforced=True, - schema_version="ovk.evidence.v2", - ) + routing_plan: AuthoritativeRoutingPlan, -def _run_enforced_lane( - *, - lane: str, - data: dict[str, Any], repo: str, - head_sha: str, - base_sha: str | None, - policy: dict[str, Any] | None, - registry_builder, - compiler, -) -> VerificationEvidence: - """Execute a lane through registry selection; result is authoritative.""" - registry = registry_builder() - try: - obligation = compiler( - data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - policy=policy, - ) - except TypeError: - obligation = compiler( - data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - ) - budget = execution_budget_from_policy(policy) - routing_config = routing_config_from_policy(policy) - context = ExecutionContext( - subject=obligation.subject, - budget=budget, - policy_digest=obligation.policy_digest, - metadata={"enforced": True, "lane": lane}, - ) - routing = route_obligation( - obligation, - registry, - context=context, - config=RoutingConfig( - mode="enforced", - strategy=routing_config.strategy, - aggregation=routing_config.aggregation, - max_selected_backends=routing_config.max_selected_backends, - prefer_deterministic=routing_config.prefer_deterministic, - allow_fallback=routing_config.allow_fallback, - accept_partial_primary=routing_config.accept_partial_primary, - enforced_lanes=frozenset({lane}), - ), - policy=policy, - ) - record = _control_plane().execute(obligation, routing, registry=registry) - return execution_record_to_evidence( - record, - author_type=str(data.get("author_type", "unknown")), - agent=str(data.get("agent", "unknown")), - task=str(data.get("task", "unknown")), - routing_enforced=True, - schema_version="ovk.evidence.v2", - ) - -def _run_enforced_infrastructure( - *, - data: dict[str, Any], - repo: str, head_sha: str, - base_sha: str | None, - policy: dict[str, Any] | None, -) -> VerificationEvidence: - return _run_enforced_lane( - lane="infrastructure", - data=data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - policy=policy, - registry_builder=build_infrastructure_registry, - compiler=compile_infrastructure_obligation, - ) - - -def _run_enforced_ci_secrets( - *, - data: dict[str, Any], - repo: str, - head_sha: str, - base_sha: str | None, - policy: dict[str, Any] | None, -) -> VerificationEvidence: - return _run_enforced_lane( - lane="ci_secrets", - data=data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - policy=policy, - registry_builder=build_ci_secrets_registry, - compiler=compile_ci_secrets_obligation, - ) - -def _run_enforced_deployment( - *, - data: dict[str, Any], - repo: str, - head_sha: str, base_sha: str | None, - policy: dict[str, Any] | None, -) -> VerificationEvidence: - return _run_enforced_lane( - lane="deployment", - data=data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - policy=policy, - registry_builder=build_deployment_registry, - compiler=compile_deployment_obligation, - ) - -def _evaluate_obligation( - obligation: dict[str, Any], - *, - routing_by_intent: dict[str, dict[str, Any]], - repo: str, - head_sha: str, - base_sha: str | None, cache_dir: Path | None, + use_cache: bool, + policy: dict[str, Any] | None = None, + + evidence_schema_version: str = "ovk.evidence.v3", + ) -> VerificationEvidence: + lane = str(obligation["lane"]) + data = obligation["input"] - intent_id = str(obligation.get("intent_id") or LANE_TO_INTENT.get(lane, lane)) + + intent_id = intent_id_for_obligation(obligation) + input_format = str(obligation.get("input_format", "infra")) + key = cache_key( + lane, + data, + subject={"repo": repo, "head_sha": head_sha, **({"base_sha": base_sha} if base_sha else {})}, + execution_fingerprint={"intent_id": intent_id, "input_format": input_format}, + ) + + precomputed_routing = routing_plan.routing_by_intent.get(intent_id) + + precomputed_typed = routing_plan.typed_obligations.get(intent_id) + + + if use_cache and cache_dir is not None: + cached = get_cached_evidence(cache_dir, key) + if cached is not None: + evidence = VerificationEvidence.model_validate(cached) + return _attach_execution_metadata( + evidence, + lane=lane, + data=data, - routing=routing_by_intent.get(intent_id), + + routing=precomputed_routing, + intent_id=intent_id, + job_id=obligation.get("job_id"), - input_format=str(obligation.get("input_format", "infra")), + + input_format=input_format, + + routing_enforced=routing_enforced_for_lane(policy, lane), + ) - # Vertical slices: enforced control plane is authoritative. - enforced_runners = { - "authorization": _run_enforced_authorization, - "self_protection": _run_enforced_self_protection, - "infrastructure": _run_enforced_infrastructure, - "ci_secrets": _run_enforced_ci_secrets, - "deployment": _run_enforced_deployment, - } - if lane in enforced_runners and routing_enforced_for_lane(policy, lane): - evidence = enforced_runners[lane]( + + + if routing_enforced_for_lane(policy, lane): + + routing_decision = require_routing_decision( + + precomputed_routing, + + intent_id=intent_id, + + lane=lane, + + policy=policy, + + ) + + if precomputed_typed is None: + + raise RuntimeError(f"missing typed obligation for enforced intent {intent_id!r}") + + evidence = _run_enforced_with_routing( + + lane=lane, + data=data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, + + routing=routing_decision, + + typed_obligation=precomputed_typed, + policy=policy, + + schema_version=evidence_schema_version, + ) + if use_cache and cache_dir is not None: + store_cached_evidence(cache_dir, key, evidence.model_dump(mode="json")) + return _attach_execution_metadata( + evidence, + lane=lane, + data=data, - routing=routing_by_intent.get(intent_id), + + routing=routing_decision, + intent_id=intent_id, + job_id=obligation.get("job_id"), - input_format=str(obligation.get("input_format", "infra")), + + input_format=input_format, + + routing_enforced=True, + ) - # Legacy path remains authoritative for non-enforced lanes. + + evidence = evaluate_lane( + lane, + data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, - input_format=str(obligation.get("input_format", "infra")), + + input_format=input_format, + policy_path=Path(obligation["policy_path"]) if obligation.get("policy_path") else None, + ) + + shadow_comparison: dict[str, Any] | None = None + routing_config = resolve_routing_config(policy) - if routing_config.mode in {"shadow", "enforced"} and lane in { - "self_protection", - "authorization", - "infrastructure", - "ci_secrets", - "deployment", - }: + + if routing_config.mode in {"shadow", "enforced"} and lane in _LANE_REGISTRY_BUILDERS: + shadow = _run_shadow_path( + lane=lane, + data=data, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + intent_id=intent_id, + policy=policy, + ) + if shadow and "record" in shadow: + legacy_status, legacy_recommendation = _legacy_status_and_recommendation(evidence) + shadow_comparison = compare_shadow_to_legacy( + shadow=shadow["record"], + legacy_status=legacy_status, + legacy_recommendation=legacy_recommendation, + ) + shadow_comparison["routing_mode"] = routing_config.mode + shadow_comparison["legacy_authoritative"] = True + elif shadow and "error" in shadow: + shadow_comparison = { + "kind": "shadow_comparison", + "agreement": False, + "error": shadow["error"], + "legacy_authoritative": True, + "routing_mode": routing_config.mode, + } + + if use_cache and cache_dir is not None: + store_cached_evidence(cache_dir, key, evidence.model_dump(mode="json")) + return _attach_execution_metadata( + evidence, + lane=lane, + data=data, - routing=routing_by_intent.get(intent_id), + + routing=precomputed_routing, + shadow_comparison=shadow_comparison, + intent_id=intent_id, + job_id=obligation.get("job_id"), - input_format=str(obligation.get("input_format", "infra")), + + input_format=input_format, + + routing_enforced=False, + ) + + + def execute_obligations( + obligations: list[dict[str, Any]], - routing_by_intent: dict[str, dict[str, Any]], + + routing_by_intent: Mapping[str, RoutingDecision | Mapping[str, Any] | None] | None = None, + *, + repo: str, + head_sha: str, + base_sha: str | None = None, + cache_dir: Path | None = None, + use_cache: bool = True, + parallel: bool = True, + policy: dict[str, Any] | None = None, + + evidence_schema_version: str = "ovk.evidence.v3", + ) -> list[VerificationEvidence]: - """Evaluate obligations in parallel when configured. - When ``policy.routing.mode`` is ``shadow``, the typed control plane also runs - and a comparison artifact is attached. Legacy lane evaluation remains - authoritative for the returned evidence decision. - """ + """Evaluate obligations using one authoritative routing decision per intent.""" + if not obligations: + return [] + + + + routing_plan = ensure_authoritative_routing( + + obligations, + + routing_by_intent, + + policy=policy, + + repo=repo, + + head_sha=head_sha, + + base_sha=base_sha, + + ) + + + if parallel and len(obligations) > 1: + with ThreadPoolExecutor(max_workers=min(len(obligations), 5)) as pool: + futures = [ + pool.submit( + _evaluate_obligation, + obligation, - routing_by_intent=routing_by_intent, + + routing_plan=routing_plan, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + cache_dir=cache_dir, + use_cache=use_cache, + policy=policy, + + evidence_schema_version=evidence_schema_version, + ) + for obligation in obligations + ] + return [future.result() for future in futures] + + return [ + _evaluate_obligation( + obligation, - routing_by_intent=routing_by_intent, + + routing_plan=routing_plan, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + cache_dir=cache_dir, + use_cache=use_cache, + policy=policy, + + evidence_schema_version=evidence_schema_version, + ) + for obligation in obligations + ] + From 4fd2b13e93d01e1004583a159f1ef3a5213a3b19 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:49 -0700 Subject: [PATCH 18/26] Require evidence v3 control-plane traces and material-set digests. Cross-bind canonical material-set digests across evidence, provenance, and attestation so forged or incomplete control-plane traces fail closed under INV-021. --- ovk/core/attestation.py | 1 + ovk/core/attestation_binding.py | 8 + ovk/core/evidence_from_execution.py | 214 ++++++++++++++++++- ovk/core/evidence_invariants.py | 49 ++++- ovk/core/materials.py | 25 ++- ovk/core/models.py | 1 + ovk/core/provenance.py | 3 + schemas/verification.evidence.v3.schema.json | 124 +++++++++++ 8 files changed, 418 insertions(+), 7 deletions(-) create mode 100644 schemas/verification.evidence.v3.schema.json diff --git a/ovk/core/attestation.py b/ovk/core/attestation.py index c7a14d8..a6cb846 100644 --- a/ovk/core/attestation.py +++ b/ovk/core/attestation.py @@ -40,6 +40,7 @@ def bundle_to_statement(bundle: EvidenceBundle) -> dict[str, Any]: "decision": evidence.decision, "obligation_id": evidence.obligation_id, "routing_id": evidence.routing_id, + "material_set_digest": evidence.material_set_digest, "compiler": evidence.compiler, "coverage": evidence.coverage, "materials": evidence.materials, diff --git a/ovk/core/attestation_binding.py b/ovk/core/attestation_binding.py index 176f5fe..39c9196 100644 --- a/ovk/core/attestation_binding.py +++ b/ovk/core/attestation_binding.py @@ -68,6 +68,14 @@ def verify_bundle_statement_binding(bundle: EvidenceBundle, statement: dict[str, message="evidence and attestation routing IDs disagree (OVK-INV-020)", ) ) + stated_material_set = evidence_items[index].get("material_set_digest") + if evidence.material_set_digest and stated_material_set != evidence.material_set_digest: + issues.append( + EvidenceInvariantIssue( + path=f"predicate.verification.evidence[{index}].material_set_digest", + message="evidence and attestation material_set_digest disagree (OVK-INV-021)", + ) + ) return issues diff --git a/ovk/core/evidence_from_execution.py b/ovk/core/evidence_from_execution.py index b204b7d..414f108 100644 --- a/ovk/core/evidence_from_execution.py +++ b/ovk/core/evidence_from_execution.py @@ -1,148 +1,354 @@ """Convert control-plane execution records into VerificationEvidence.""" + + from __future__ import annotations + + from typing import Any + + from ovk.compilers.authorization import CoveragePolicy, strict_allow_permitted + from ovk.core.bundle import content_digest + from ovk.core.execution_models import ObligationExecutionRecord + +from ovk.core.materials import material_set_digest_for_obligation + from ovk.core.models import BackendClaim, MergeRecommendation, VerificationEvidence, VerificationStatus + + + def execution_record_to_evidence( + record: ObligationExecutionRecord, + *, + author_type: str = "unknown", + agent: str = "unknown", + task: str = "unknown", + routing_enforced: bool = False, + schema_version: str = "ovk.evidence.v2", + coverage_policy: CoveragePolicy | None = None, + ) -> VerificationEvidence: + """Project an obligation execution record into public evidence.""" + obligation = record.obligation + routing = record.routing + counterexamples: list[dict[str, Any]] = [] + for result in record.results: + counterexamples.extend(result.counterexamples) + + claims = [ + BackendClaim( + backend=result.backend, + guarantee_type=result.guarantee_type, + status=result.status, + assumptions=list(result.assumptions), + limits=list(result.limits), + adapter_version=next( + ( + item.adapter_version + for item in record.backend_obligations + if item.backend == result.backend + ), + None, + ), + ) + for result in sorted(record.results, key=lambda item: item.backend) + ] + if not claims: + claims = [ + BackendClaim( + backend="none", + guarantee_type="none", + status=VerificationStatus.UNKNOWN, + assumptions=["No backend produced a claim."], + limits=["Absence of backend evidence cannot allow."], + ) + ] + + + material_payloads = [item.model_dump(mode="json") for item in obligation.materials] + + material_set_digest = material_set_digest_for_obligation(obligation) + + + artifacts: list[dict[str, Any]] = [] + for result in record.results: + artifacts.extend(result.generated_artifacts) + for item in record.open_obligations: + artifacts.append(dict(item)) + artifacts.append( + { + "kind": "routing_enforced", + "value": routing_enforced, + "routing_id": routing.routing_id, + "obligation_id": obligation.obligation_id, + } + ) + + artifacts.append( + + { + + "kind": "control_plane_trace", + + "compiler": { + + "compiler_id": obligation.compiler_id, + + "compiler_version": obligation.compiler_version, + + }, + + "coverage": obligation.coverage.model_dump(mode="json"), + + "material_set_digest": material_set_digest, + + "routing_id": routing.routing_id, + + "requested_backends": list(routing.requested), + + "eligible_backends": [item.backend for item in routing.eligible], + + "selected_backends": [item.backend for item in routing.selected], + + "attempted_backends": [item.backend for item in record.attempts], + + "executed_backends": [item.backend for item in record.results], + + "execution_attempts": [item.model_dump(mode="json") for item in record.attempts], + + "routing_enforced": routing_enforced, + + "aggregation_policy": routing.aggregation_policy, + + } + + ) + if obligation.compiler_id: + artifacts.append( + { + "kind": "compiler_identity", + "compiler_id": obligation.compiler_id, + "compiler_version": obligation.compiler_version, + "coverage": obligation.coverage.model_dump(mode="json"), - "materials": [item.model_dump(mode="json") for item in obligation.materials], + + "materials": material_payloads, + + "material_set_digest": material_set_digest, + } + ) + + recommendation = record.merge_recommendation + aggregation_reason = record.aggregation_reason - # Incomplete abstraction cannot produce allow under strict coverage policy. + policy = coverage_policy or CoveragePolicy() + allow_ok = obligation.abstraction.get("strict_allow_permitted") + if allow_ok is None: + allow_ok = strict_allow_permitted(obligation.coverage, policy) + if recommendation == MergeRecommendation.ALLOW and not allow_ok: + recommendation = MergeRecommendation.REQUIRE_HUMAN_REVIEW + aggregation_reason = ( + f"{aggregation_reason}; incomplete abstraction cannot allow under strict coverage" + ) + artifacts.append( + { + "kind": "incomplete_abstraction", + "coverage": obligation.coverage.model_dump(mode="json"), + "reason": "strict allow blocked unless coverage complete or policy accepts partial", + } + ) + + decision = { + "merge_recommendation": recommendation.value, + "human_review_required": recommendation.value != "allow", + "aggregation_reason": aggregation_reason, + "routing_enforced": routing_enforced, + + "fallback_used": record.fallback_used, + + "fallback_accepted": record.fallback_accepted, + + "fallback_cause": record.fallback_cause, + } + + evidence_id = content_digest( + { + "obligation_id": obligation.obligation_id, + "routing_id": routing.routing_id, + + "material_set_digest": material_set_digest, + "results": [claim.model_dump(mode="json") for claim in claims], + } + )[:24] + + return VerificationEvidence( + evidence_id=f"ev-{evidence_id}", + schema_version=schema_version, - subject=obligation.subject.model_dump(mode="json"), + + subject={key: value for key, value in obligation.subject.model_dump(mode="json").items() if value is not None}, + change_origin={"author_type": author_type, "agent": agent, "task": task}, + intent={ + "intent_id": obligation.intent_id, + "title": obligation.intent_id, + "risk": {"severity": obligation.severity.value}, + }, + backend_claims=claims, + counterexamples=counterexamples, + generated_artifacts=artifacts, + decision=decision, + obligation_id=obligation.obligation_id, + routing_id=routing.routing_id, + + material_set_digest=material_set_digest if schema_version.endswith(".v3") else None, + compiler={ + "compiler_id": obligation.compiler_id, + "compiler_version": obligation.compiler_version, + }, - materials=[item.model_dump(mode="json") for item in obligation.materials], + + materials=material_payloads, + coverage=obligation.coverage.model_dump(mode="json"), + requested_backends=list(routing.requested), + eligible_backends=[item.backend for item in routing.eligible], + selected_backends=[item.backend for item in routing.selected], + attempted_backends=[item.backend for item in record.attempts], + executed_backends=[item.backend for item in record.results], + execution_attempts=[item.model_dump(mode="json") for item in record.attempts], + aggregation_policy=routing.aggregation_policy, + routing_enforced=routing_enforced, + ) + diff --git a/ovk/core/evidence_invariants.py b/ovk/core/evidence_invariants.py index 5ab00f7..50dea1c 100644 --- a/ovk/core/evidence_invariants.py +++ b/ovk/core/evidence_invariants.py @@ -6,6 +6,7 @@ from typing import Any from ovk.core.bundle import content_digest +from ovk.core.materials import compute_material_set_digest from ovk.core.models import EvidenceBundle, VerificationStatus @@ -230,7 +231,8 @@ def _check_control_plane_invariants(bundle: EvidenceBundle) -> list[EvidenceInva for index, evidence in enumerate(bundle.evidence): path = f"evidence[{index}]" is_v2 = str(evidence.schema_version).endswith(".v2") or evidence.routing_enforced - if not is_v2 and evidence.obligation_id is None and evidence.routing_id is None: + is_v3 = str(evidence.schema_version).endswith(".v3") + if not is_v2 and not is_v3 and evidence.obligation_id is None and evidence.routing_id is None: continue selected = list(evidence.selected_backends or []) @@ -332,6 +334,23 @@ def _check_control_plane_invariants(bundle: EvidenceBundle) -> list[EvidenceInva message="material digests must be present (OVK-INV-015)", ) ) + if is_v3 or evidence.routing_enforced: + expected_material_set = compute_material_set_digest(evidence.materials) + stated_material_set = evidence.material_set_digest + if not stated_material_set: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.material_set_digest", + message="evidence v3 must include material_set_digest (OVK-INV-021)", + ) + ) + elif stated_material_set != expected_material_set: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.material_set_digest", + message="material_set_digest must match recomputed canonical digest (OVK-INV-021)", + ) + ) coverage = evidence.coverage or {} recommendation = _decision_value(evidence.decision, "merge_recommendation") @@ -383,7 +402,33 @@ def _check_control_plane_invariants(bundle: EvidenceBundle) -> list[EvidenceInva ) ) - # OVK-INV-020 checked at attestation binding time; also flag missing routing_id on enforced. + if is_v3 or evidence.routing_enforced: + trace_artifacts = [a for a in evidence.generated_artifacts if a.get("kind") == "control_plane_trace"] + if not trace_artifacts and is_v3: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.generated_artifacts", + message="evidence v3 must include control_plane_trace artifact (OVK-INV-022)", + ) + ) + elif trace_artifacts: + trace = trace_artifacts[0] + if trace.get("routing_id") not in {None, evidence.routing_id}: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.generated_artifacts", + message="control_plane_trace routing_id must match evidence routing_id (OVK-INV-022)", + ) + ) + trace_digest = trace.get("material_set_digest") + if is_v3 and trace_digest and trace_digest != evidence.material_set_digest: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.generated_artifacts", + message="control_plane_trace material_set_digest must match evidence (OVK-INV-021)", + ) + ) + if evidence.routing_enforced and not evidence.routing_id: issues.append( EvidenceInvariantIssue( diff --git a/ovk/core/materials.py b/ovk/core/materials.py index 4223e71..3e84ddc 100644 --- a/ovk/core/materials.py +++ b/ovk/core/materials.py @@ -6,7 +6,7 @@ from typing import Any, cast from ovk.core.bundle import content_digest -from ovk.core.execution_models import MaterialKind, MaterialReference +from ovk.core.execution_models import MaterialKind, MaterialReference, VerificationObligation def canonical_material_bytes(payload: Any) -> bytes: @@ -19,6 +19,29 @@ def canonical_material_bytes(payload: Any) -> bytes: ).encode("utf-8") +def compute_material_set_digest(materials: list[Any] | None) -> str: + """Compute a canonical digest over sorted material ids and content digests.""" + entries: list[dict[str, str]] = [] + for item in materials or []: + if hasattr(item, "model_dump"): + payload = item.model_dump(mode="json") + elif isinstance(item, dict): + payload = item + else: + continue + material_id = str(payload.get("material_id") or payload.get("id") or "") + digest = str(payload.get("sha256") or payload.get("digest") or "") + if material_id or digest: + entries.append({"material_id": material_id, "sha256": digest}) + entries.sort(key=lambda row: (row["material_id"], row["sha256"])) + return content_digest({"materials": entries}) + + +def material_set_digest_for_obligation(obligation: VerificationObligation) -> str: + """Return the canonical material-set digest for one typed obligation.""" + return compute_material_set_digest([item.model_dump(mode="json") for item in obligation.materials]) + + def material_reference_from_payload( *, material_id: str, diff --git a/ovk/core/models.py b/ovk/core/models.py index 4f0e346..d0940cb 100644 --- a/ovk/core/models.py +++ b/ovk/core/models.py @@ -96,6 +96,7 @@ class VerificationEvidence(BaseModel): routing_id: str | None = None compiler: dict[str, Any] | None = None materials: list[dict[str, Any]] | None = None + material_set_digest: str | None = None coverage: dict[str, Any] | None = None requested_backends: list[str] | None = None eligible_backends: list[str] | None = None diff --git a/ovk/core/provenance.py b/ovk/core/provenance.py index cc59bdc..2275261 100644 --- a/ovk/core/provenance.py +++ b/ovk/core/provenance.py @@ -87,6 +87,9 @@ def build_provenance_statement( "control_plane": { "obligation_ids": [item.obligation_id for item in bundle.evidence if item.obligation_id], "routing_ids": [item.routing_id for item in bundle.evidence if item.routing_id], + "material_set_digests": [ + item.material_set_digest for item in bundle.evidence if item.material_set_digest + ], "routing_enforced": [bool(item.routing_enforced) for item in bundle.evidence], "compilers": [item.compiler for item in bundle.evidence if item.compiler], "coverage": [item.coverage for item in bundle.evidence if item.coverage], diff --git a/schemas/verification.evidence.v3.schema.json b/schemas/verification.evidence.v3.schema.json new file mode 100644 index 0000000..b17a6fb --- /dev/null +++ b/schemas/verification.evidence.v3.schema.json @@ -0,0 +1,124 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openverification.dev/schemas/verification.evidence.v3.schema.json", + "title": "Verification Evidence v3", + "type": "object", + "required": [ + "evidence_id", + "schema_version", + "subject", + "intent", + "backend_claims", + "decision", + "obligation_id", + "routing_id", + "material_set_digest", + "compiler", + "materials", + "coverage", + "requested_backends", + "eligible_backends", + "selected_backends", + "attempted_backends", + "executed_backends", + "execution_attempts", + "aggregation_policy", + "routing_enforced" + ], + "properties": { + "evidence_id": { "type": "string", "minLength": 1 }, + "schema_version": { "type": "string", "const": "ovk.evidence.v3" }, + "subject": { + "type": "object", + "required": ["repo", "head_sha"], + "properties": { + "repo": { "type": "string" }, + "pull_request": { "type": ["integer", "string", "null"] }, + "head_sha": { "type": "string" }, + "base_sha": { "type": ["string", "null"] } + }, + "additionalProperties": true + }, + "change_origin": { "type": "object", "additionalProperties": true }, + "intent": { + "type": "object", + "required": ["intent_id", "title"], + "properties": { + "intent_id": { "type": "string" }, + "title": { "type": "string" } + }, + "additionalProperties": true + }, + "backend_claims": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["backend", "guarantee_type", "status"], + "properties": { + "backend": { "type": "string" }, + "guarantee_type": { "type": "string" }, + "status": { "type": "string", "enum": ["pass", "fail", "unknown", "error", "skipped"] }, + "assumptions": { "type": "array", "items": { "type": "string" } }, + "limits": { "type": "array", "items": { "type": "string" } }, + "tool_version": { "type": ["string", "null"] }, + "adapter_version": { "type": ["string", "null"] } + }, + "additionalProperties": true + } + }, + "counterexamples": { "type": "array", "items": { "type": "object", "additionalProperties": true } }, + "generated_artifacts": { "type": "array", "items": { "type": "object", "additionalProperties": true } }, + "decision": { + "type": "object", + "required": ["merge_recommendation", "routing_enforced", "aggregation_reason"], + "properties": { + "merge_recommendation": { + "type": "string", + "enum": ["allow", "block", "require_human_review", "allow_with_warning", "require_stronger_check"] + }, + "human_review_required": { "type": "boolean" }, + "routing_enforced": { "type": "boolean" }, + "aggregation_reason": { "type": "string" }, + "fallback_used": { "type": "boolean" }, + "fallback_accepted": { "type": "boolean" }, + "fallback_cause": { "type": ["string", "null"] } + }, + "additionalProperties": true + }, + "obligation_id": { "type": "string", "minLength": 1 }, + "routing_id": { "type": "string", "minLength": 1 }, + "material_set_digest": { "type": "string", "minLength": 1 }, + "compiler": { + "type": "object", + "required": ["compiler_id", "compiler_version"], + "properties": { + "compiler_id": { "type": "string" }, + "compiler_version": { "type": "string" } + }, + "additionalProperties": true + }, + "materials": { + "type": "array", + "items": { + "type": "object", + "required": ["material_id", "sha256"], + "properties": { + "material_id": { "type": "string" }, + "sha256": { "type": "string" } + }, + "additionalProperties": true + } + }, + "coverage": { "type": "object", "additionalProperties": true }, + "requested_backends": { "type": "array", "items": { "type": "string" } }, + "eligible_backends": { "type": "array", "items": { "type": "string" } }, + "selected_backends": { "type": "array", "items": { "type": "string" } }, + "attempted_backends": { "type": "array", "items": { "type": "string" } }, + "executed_backends": { "type": "array", "items": { "type": "string" } }, + "execution_attempts": { "type": "array", "items": { "type": "object", "additionalProperties": true } }, + "aggregation_policy": { "type": "string", "minLength": 1 }, + "routing_enforced": { "type": "boolean" } + }, + "additionalProperties": true +} From 51d19b7065eb9fa492c09da5c3a3abb0abef3673 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:49 -0700 Subject: [PATCH 19/26] Add adversarial tests for coverage, fallback, and trust provenance. Lock fail-closed behavior for unmet coverage, blocked fallback terminations, and untrusted self-protection metadata before later routing and isolation layers. --- tests/test_trust_chain_r2_pr3_pr4_pr5.py | 320 +++++++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 tests/test_trust_chain_r2_pr3_pr4_pr5.py diff --git a/tests/test_trust_chain_r2_pr3_pr4_pr5.py b/tests/test_trust_chain_r2_pr3_pr4_pr5.py new file mode 100644 index 0000000..2a8455e --- /dev/null +++ b/tests/test_trust_chain_r2_pr3_pr4_pr5.py @@ -0,0 +1,320 @@ +"""Adversarial tests for OVK R2 trust-chain PR3–PR5.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from ovk.adapters.authorization import build_authorization_registry +from ovk.core.authorization_compiler import compile_authorization_obligation +from ovk.core.backend_aggregation import ( + FALLBACK_BLOCKING_TERMINATIONS, + aggregate_fail_dominant_v1, + evaluate_fallback_acceptance, +) +from ovk.core.backend_control_plane import BackendControlPlane +from ovk.core.execution_models import ( + BackendCapabilityAssessment, + BackendObligation, + BackendSelection, + ExecutionAttempt, + ExecutionBudget, + ExecutionContext, + FallbackPolicy, + NormalizedBackendResult, + RoutingDecision, +) +from ovk.core.models import MergeRecommendation, VerificationStatus +from ovk.core.router import RoutingConfig, route_obligation, select_primary_with_optional_corroboration +from ovk.core.self_protection_compiler import ( + compile_self_protection_obligation, + resolve_metadata_trusted, +) + + +def _budget() -> ExecutionBudget: + return ExecutionBudget( + total_wall_time_seconds=60, + per_backend_wall_time_seconds=30, + max_memory_mb=512, + max_parallel_backends=2, + allow_network=False, + allow_repository_write=False, + ) + + +def _assessment( + *, + backend: str = "test-backend", + support: str = "supported", + coverage_requirements_met: bool = True, + score: float = 0.9, +) -> BackendCapabilityAssessment: + return BackendCapabilityAssessment( + backend=backend, + support=support, # type: ignore[arg-type] + score=score, + guarantee_type="policy_evaluation", + material_requirements_met=True, + coverage_requirements_met=coverage_requirements_met, + native_available=False, + estimated_wall_time_seconds=5.0, + estimated_memory_mb=256, + reasons=["test assessment"], + ) + + +def test_incomplete_coverage_rejected_as_required_primary() -> None: + assessments = [_assessment(coverage_requirements_met=False)] + selected, rejected, eligible = select_primary_with_optional_corroboration( + assessments, + acceptable_guarantees=["policy_evaluation"], + config=RoutingConfig(accept_partial_primary=False), + budget=_budget(), + ) + assert selected == [] + assert any(item.reason == "coverage requirements not met" for item in rejected) + assert eligible == [] + + +def test_incomplete_coverage_eligible_optional_when_policy_allows() -> None: + assessments = [_assessment(coverage_requirements_met=False)] + selected, rejected, eligible = select_primary_with_optional_corroboration( + assessments, + acceptable_guarantees=["policy_evaluation"], + config=RoutingConfig(accept_partial_primary=True, max_selected_backends=2), + budget=_budget(), + ) + assert not any(item.required for item in selected) + assert rejected == [] + assert len(eligible) == 1 + assert eligible[0].support == "partial" + assert "incomplete coverage" in " ".join(eligible[0].reasons) + if selected: + assert all(not item.required for item in selected) + + +def test_guarantee_mismatch_fails_closed_without_rewrite() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_protected.json").read_text(encoding="utf-8")) + registry = build_authorization_registry() + obligation = compile_authorization_obligation(data, repo="r", head_sha="h") + budget = _budget() + routing = route_obligation( + obligation, + registry, + context=ExecutionContext(subject=obligation.subject, budget=budget, policy_digest="p"), + config=RoutingConfig(prefer_deterministic=True, max_selected_backends=1), + policy={"budget": {"allowed_backends": ["authorization-deterministic"]}}, + ) + routing = routing.model_copy( + update={ + "selected": [ + BackendSelection( + backend="authorization-deterministic", + reason="test", + expected_guarantee="smt_refutation_search", + required=True, + ) + ] + } + ) + + adapter = registry.require("authorization-deterministic") + original_compile = adapter.compile + + def _mismatching_compile( + compiled_obligation: Any, + compiled_routing: RoutingDecision, + ) -> BackendObligation: + compiled = original_compile(compiled_obligation, compiled_routing) + return compiled.model_copy(update={"expected_guarantee": "deterministic_witness"}) + + adapter.compile = _mismatching_compile # type: ignore[method-assign] + + record = BackendControlPlane(use_hardened_cache=False).execute( + obligation, + routing, + registry=registry, + cache=None, + ) + assert record.results[0].status == VerificationStatus.UNKNOWN + assert record.attempts[0].termination == "invalid_output" + assert record.merge_recommendation == MergeRecommendation.REQUIRE_HUMAN_REVIEW + assert "compiler guarantee mismatch" in record.results[0].limits[0] + + +@pytest.mark.parametrize("termination", sorted(FALLBACK_BLOCKING_TERMINATIONS)) +def test_blocking_terminations_never_accept_fallback(termination: str) -> None: + policy = FallbackPolicy( + allow_fallback=True, + fallback_backends=["authorization-deterministic"], + acceptable_fallback_guarantees=["deterministic_witness"], + ) + used, accepted, cause = evaluate_fallback_acceptance( + policy=policy, + selected=[ + BackendSelection( + backend="authorization-deterministic", + reason="test", + expected_guarantee="smt_refutation_search", + required=True, + ) + ], + attempts=[ + ExecutionAttempt( + attempt_id="att-1", + backend_obligation_id="bo-1", + backend="authorization-deterministic", + required=True, + started_at="2026-01-01T00:00:00Z", + finished_at="2026-01-01T00:00:01Z", + duration_ms=1.0, + termination=termination, # type: ignore[arg-type] + native_execution=False, + ) + ], + results=[ + NormalizedBackendResult( + attempt_id="att-1", + backend="authorization-deterministic", + status=VerificationStatus.PASS, + guarantee_type="deterministic_witness", + ) + ], + acceptable_guarantees=["smt_refutation_search"], + ) + assert used is True + assert accepted is False + assert cause == termination + + +def test_allow_fallback_without_blocking_cause_can_accept() -> None: + policy = FallbackPolicy( + allow_fallback=True, + fallback_backends=["authorization-deterministic"], + acceptable_fallback_guarantees=["deterministic_witness"], + ) + used, accepted, _ = evaluate_fallback_acceptance( + policy=policy, + selected=[ + BackendSelection( + backend="authorization-deterministic", + reason="test", + expected_guarantee="smt_refutation_search", + required=True, + ) + ], + attempts=[ + ExecutionAttempt( + attempt_id="att-1", + backend_obligation_id="bo-1", + backend="authorization-deterministic", + required=True, + started_at="2026-01-01T00:00:00Z", + finished_at="2026-01-01T00:00:01Z", + duration_ms=1.0, + termination="completed", + native_execution=False, + ) + ], + results=[ + NormalizedBackendResult( + attempt_id="att-1", + backend="authorization-deterministic", + status=VerificationStatus.PASS, + guarantee_type="deterministic_witness", + ) + ], + acceptable_guarantees=["smt_refutation_search"], + ) + assert used is True + assert accepted is True + + +def test_aggregate_fail_dominant_rejects_unaccepted_fallback_pass() -> None: + outcome = aggregate_fail_dominant_v1( + obligation_id="obl", + selected=[ + BackendSelection( + backend="a", + reason="test", + expected_guarantee="smt_refutation_search", + required=True, + ) + ], + results=[ + NormalizedBackendResult( + attempt_id="att-1", + backend="a", + status=VerificationStatus.PASS, + guarantee_type="deterministic_witness", + ) + ], + acceptable_guarantees=["smt_refutation_search"], + fallback_policy=FallbackPolicy(allow_fallback=False), + attempts=[ + ExecutionAttempt( + attempt_id="att-1", + backend_obligation_id="bo-1", + backend="a", + required=True, + started_at="2026-01-01T00:00:00Z", + finished_at="2026-01-01T00:00:01Z", + duration_ms=1.0, + termination="timeout", + native_execution=False, + ) + ], + ) + assert outcome.merge_recommendation == MergeRecommendation.REQUIRE_STRONGER_CHECK + assert outcome.fallback_used is True + assert outcome.fallback_accepted is False + assert outcome.fallback_cause == "timeout" + + +def test_metadata_trusted_defaults_false() -> None: + obligation = compile_self_protection_obligation( + { + "before": {"required_checks": ["ovk-verify"]}, + "after": {"required_checks": ["ovk-verify"]}, + }, + repo="r", + head_sha="h", + base_sha="b", + ) + assert obligation.abstraction["metadata_trusted"] is False + branch_materials = [item for item in obligation.materials if item.kind == "branch_protection"] + assert branch_materials + assert all(not item.trusted for item in branch_materials) + + +def test_resolve_metadata_trusted_requires_provenance_kind() -> None: + assert resolve_metadata_trusted(None) is False + assert resolve_metadata_trusted({}) is False + assert resolve_metadata_trusted({"trust": {"metadata_trusted": True}}) is False + assert ( + resolve_metadata_trusted( + { + "trust": { + "metadata_trusted": True, + "provenance_kind": "protected_base_workflow", + } + } + ) + is True + ) + + +def test_current_state_only_routing_flag_cannot_authorize_trust() -> None: + assert ( + resolve_metadata_trusted( + { + "routing": {"metadata_trusted": True}, + "trust": {"metadata_trusted": True}, + } + ) + is False + ) From b121ca3bb7c04b2705408b3c3bc47e8c9562c1b1 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:49 -0700 Subject: [PATCH 20/26] Cover routing, worker isolation, and evidence v3 with regression tests. Assert single-route identity, deterministic/native worker isolation, and v3 schema expectations on enforced lanes so trust-chain PRs 6-9 stay fail-closed. --- tests/test_adapter_isolation_r2_pr8.py | 75 ++++++++++++ tests/test_authorization_enforcement.py | 2 +- tests/test_deterministic_worker_r2_pr7.py | 104 ++++++++++++++++ tests/test_evidence_v3_r2_pr9.py | 108 +++++++++++++++++ tests/test_remaining_lane_enforcement.py | 2 +- tests/test_routing_pipeline_r2_pr6.py | 141 ++++++++++++++++++++++ tests/test_self_protection_enforcement.py | 2 +- 7 files changed, 431 insertions(+), 3 deletions(-) create mode 100644 tests/test_adapter_isolation_r2_pr8.py create mode 100644 tests/test_deterministic_worker_r2_pr7.py create mode 100644 tests/test_evidence_v3_r2_pr9.py create mode 100644 tests/test_routing_pipeline_r2_pr6.py diff --git a/tests/test_adapter_isolation_r2_pr8.py b/tests/test_adapter_isolation_r2_pr8.py new file mode 100644 index 0000000..c4f8d84 --- /dev/null +++ b/tests/test_adapter_isolation_r2_pr8.py @@ -0,0 +1,75 @@ +"""PR8 — native adapter worker isolation tests.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import patch + +from ovk.adapters.authorization.z3_adapter import Z3NativeAuthorizationAdapter +from ovk.adapters.self_protection.opa_adapter import OpaNativeSelfProtectionAdapter +from ovk.core.execution_budget import LocalSubprocessWorker, WorkerResult +from ovk.core.execution_models import ExecutionBudget + + +def _budget() -> ExecutionBudget: + return ExecutionBudget( + total_wall_time_seconds=30, + per_backend_wall_time_seconds=30, + max_memory_mb=512, + max_parallel_backends=1, + allow_network=False, + allow_repository_write=False, + ) + + +def test_z3_adapter_requires_worker() -> None: + adapter = Z3NativeAuthorizationAdapter() + from ovk.core.authorization_compiler import compile_authorization_obligation + from ovk.core.routing_pipeline import route_compiled_obligation + + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + obligation = compile_authorization_obligation(data, repo="r", head_sha="h") + routing = route_compiled_obligation(obligation, lane="authorization") + compiled = adapter.compile(obligation, routing) + raw = adapter.run(compiled, _budget(), worker=None) + assert raw.termination == "tool_error" + + +def test_opa_adapter_requires_worker() -> None: + adapter = OpaNativeSelfProtectionAdapter() + from ovk.core.self_protection_compiler import compile_self_protection_obligation + from ovk.core.routing_pipeline import route_compiled_obligation + + data = {"before": {"required_check": "ovk"}, "after": {"required_check": None}} + obligation = compile_self_protection_obligation(data, repo="r", head_sha="h", metadata_trusted=True) + routing = route_compiled_obligation(obligation, lane="self_protection") + compiled = adapter.compile(obligation, routing) + raw = adapter.run(compiled, _budget(), worker=None) + assert raw.termination == "tool_error" + + +def test_worker_timeout_enforced_for_z3_not_post_hoc() -> None: + adapter = Z3NativeAuthorizationAdapter() + from ovk.core.authorization_compiler import compile_authorization_obligation + from ovk.core.routing_pipeline import route_compiled_obligation + + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + obligation = compile_authorization_obligation(data, repo="r", head_sha="h") + routing = route_compiled_obligation(obligation, lane="authorization") + compiled = adapter.compile(obligation, routing) + + class SlowWorker: + def run(self, command, *, cwd, env=None, timeout_seconds=30.0, max_stdout_bytes=1_000_000, max_stderr_bytes=1_000_000): + return WorkerResult( + exit_code=None, + timed_out=True, + stdout="", + stderr="timed out", + cwd=str(cwd), + command=tuple(command), + ) + + raw = adapter.run(compiled, _budget(), worker=SlowWorker()) + assert raw.termination == "timeout" + assert raw.raw_result.get("reason") == "worker execution timed out" diff --git a/tests/test_authorization_enforcement.py b/tests/test_authorization_enforcement.py index 08d87b2..1cace6a 100644 --- a/tests/test_authorization_enforcement.py +++ b/tests/test_authorization_enforcement.py @@ -101,7 +101,7 @@ def test_enforced_authorization_emits_v2_preview_fields() -> None: ) evidence = evidence_items[0] assert evidence.routing_enforced is True - assert evidence.schema_version == "ovk.evidence.v2" + assert evidence.schema_version == "ovk.evidence.v3" assert evidence.obligation_id assert evidence.routing_id assert evidence.selected_backends diff --git a/tests/test_deterministic_worker_r2_pr7.py b/tests/test_deterministic_worker_r2_pr7.py new file mode 100644 index 0000000..1987837 --- /dev/null +++ b/tests/test_deterministic_worker_r2_pr7.py @@ -0,0 +1,104 @@ +"""PR7 — isolated deterministic worker tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from ovk.adapters.authorization.deterministic_adapter import AuthorizationDeterministicAdapter +from ovk.core.backend_control_plane import BackendControlPlane +from ovk.core.deterministic_evaluators import evaluate_deterministic +from ovk.core.execution_budget import LocalSubprocessWorker +from ovk.core.execution_models import ExecutionBudget +from ovk.core.worker_runner import run_evaluator_in_worker + + +def test_deterministic_adapter_requires_worker() -> None: + adapter = AuthorizationDeterministicAdapter() + obligation = adapter.compile( + __import__("ovk.core.authorization_compiler", fromlist=["compile_authorization_obligation"]).compile_authorization_obligation( + json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")), + repo="r", + head_sha="h", + ), + __import__("ovk.core.routing_pipeline", fromlist=["route_compiled_obligation"]).route_compiled_obligation( + __import__("ovk.core.authorization_compiler", fromlist=["compile_authorization_obligation"]).compile_authorization_obligation( + json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")), + repo="r", + head_sha="h", + ), + lane="authorization", + ), + ) + budget = ExecutionBudget( + total_wall_time_seconds=30, + per_backend_wall_time_seconds=30, + max_memory_mb=512, + max_parallel_backends=1, + allow_network=False, + allow_repository_write=False, + ) + raw = adapter.run(obligation, budget, worker=None) + assert raw.termination == "tool_error" + assert raw.stderr and ("BackendWorker" in raw.stderr or "missing worker" in raw.stderr) + + +def test_deterministic_worker_runs_in_subprocess() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + worker = LocalSubprocessWorker() + outcome = run_evaluator_in_worker( + worker, + evaluator_id="authorization-deterministic", + payload={"input": data, "mode": "deterministic"}, + timeout_seconds=30.0, + ) + assert not outcome.timed_out + assert not outcome.worker_rejected + assert outcome.raw_result["status"] == "fail" + + +def test_zero_budget_rejected_before_worker_spawn() -> None: + worker = LocalSubprocessWorker() + outcome = run_evaluator_in_worker( + worker, + evaluator_id="authorization-deterministic", + payload={"input": {}}, + timeout_seconds=0.0, + ) + assert outcome.termination == "timeout" + assert outcome.worker_rejected + + +def test_control_plane_passes_worker_to_deterministic_adapter() -> None: + from ovk.adapters.authorization import build_authorization_registry + from ovk.core.authorization_compiler import compile_authorization_obligation + from ovk.core.routing_pipeline import route_compiled_obligation + + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + obligation = compile_authorization_obligation(data, repo="r", head_sha="h") + routing = route_compiled_obligation( + obligation, + lane="authorization", + policy={"budget": {"allowed_backends": ["authorization-deterministic"]}}, + ) + record = BackendControlPlane(worker=LocalSubprocessWorker()).execute( + obligation, + routing, + registry=build_authorization_registry(), + ) + assert record.results + assert record.results[0].status.value == "fail" + + +def test_evaluator_registry_covers_all_deterministic_backends() -> None: + for evaluator_id in ( + "authorization-deterministic", + "self-protection-deterministic", + "infrastructure-deterministic", + "ci-secrets-deterministic", + "deployment-deterministic", + ): + result = evaluate_deterministic(evaluator_id, {"input": {}}) + assert "raw_result" in result diff --git a/tests/test_evidence_v3_r2_pr9.py b/tests/test_evidence_v3_r2_pr9.py new file mode 100644 index 0000000..f753e1f --- /dev/null +++ b/tests/test_evidence_v3_r2_pr9.py @@ -0,0 +1,108 @@ +"""PR9 — evidence v3 and material-set binding tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from ovk.core.adapter_runtime import execute_obligations +from ovk.core.attestation import bundle_to_statement +from ovk.core.attestation_binding import verify_bundle_statement_binding +from ovk.core.bundle import make_bundle +from ovk.core.evidence_invariants import check_evidence_bundle_invariants +from ovk.core.materials import compute_material_set_digest +from ovk.core.provenance import build_provenance_statement +from ovk.core.schema_validation import load_json, validate_against_schema +from ovk.paths import schema_path + + +def _auth_policy(): + return { + "routing": { + "enforced_lanes": ["authorization"], + "prefer_deterministic": True, + }, + "budget": {"allowed_backends": ["authorization-deterministic"]}, + } + + +def test_compute_material_set_digest_is_order_insensitive() -> None: + materials_a = [ + {"material_id": "b", "sha256": "2" * 64}, + {"material_id": "a", "sha256": "1" * 64}, + ] + materials_b = list(reversed(materials_a)) + assert compute_material_set_digest(materials_a) == compute_material_set_digest(materials_b) + + +def test_enforced_emission_uses_evidence_v3_with_material_set_digest() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + evidence_items = execute_obligations( + [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}], + {}, + repo="example/repo", + head_sha="abc", + use_cache=False, + policy=_auth_policy(), + evidence_schema_version="ovk.evidence.v3", + ) + evidence = evidence_items[0] + assert evidence.schema_version == "ovk.evidence.v3" + assert evidence.material_set_digest + assert evidence.material_set_digest == compute_material_set_digest(evidence.materials) + + +def test_evidence_v3_schema_validation() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + evidence = execute_obligations( + [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}], + {}, + repo="example/repo", + head_sha="abc", + use_cache=False, + policy=_auth_policy(), + evidence_schema_version="ovk.evidence.v3", + )[0] + schema = load_json(schema_path("verification.evidence.v3.schema.json")) + report = validate_against_schema(evidence.model_dump(mode="json"), schema) + assert report.valid, [issue.message for issue in report.issues] + + +def test_v3_invariants_and_attestation_material_binding() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + evidence = execute_obligations( + [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}], + {}, + repo="example/repo", + head_sha="abc", + use_cache=False, + policy=_auth_policy(), + evidence_schema_version="ovk.evidence.v3", + )[0] + bundle = make_bundle([evidence]) + issues = check_evidence_bundle_invariants(bundle) + assert not [issue for issue in issues if issue.severity == "error"] + + statement = bundle_to_statement(bundle) + binding_issues = verify_bundle_statement_binding(bundle, statement) + assert not binding_issues + + provenance = build_provenance_statement(bundle) + assert evidence.material_set_digest in provenance["control_plane"]["material_set_digests"] + + +def test_adversarial_material_set_digest_mismatch_fails_closed() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + evidence = execute_obligations( + [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}], + {}, + repo="example/repo", + head_sha="abc", + use_cache=False, + policy=_auth_policy(), + evidence_schema_version="ovk.evidence.v3", + )[0] + tampered = evidence.model_copy(update={"material_set_digest": "deadbeef"}) + bundle = make_bundle([tampered]) + issues = check_evidence_bundle_invariants(bundle) + assert any("material_set_digest" in issue.message for issue in issues) diff --git a/tests/test_remaining_lane_enforcement.py b/tests/test_remaining_lane_enforcement.py index 556e31e..c0c26e0 100644 --- a/tests/test_remaining_lane_enforcement.py +++ b/tests/test_remaining_lane_enforcement.py @@ -254,7 +254,7 @@ def test_enforced_infrastructure_blocks_public_sensitive() -> None: ) evidence = evidence_items[0] assert evidence.routing_enforced is True - assert evidence.schema_version == "ovk.evidence.v2" + assert evidence.schema_version == "ovk.evidence.v3" assert evidence.decision.get("merge_recommendation") == "block" assert "infrastructure-deterministic" in (evidence.selected_backends or []) diff --git a/tests/test_routing_pipeline_r2_pr6.py b/tests/test_routing_pipeline_r2_pr6.py new file mode 100644 index 0000000..0b1d94f --- /dev/null +++ b/tests/test_routing_pipeline_r2_pr6.py @@ -0,0 +1,141 @@ +"""PR6 — single authoritative routing pipeline tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from ovk.core.adapter_runtime import execute_obligations +from ovk.core.kernel import execute_kernel +from ovk.core.routing_pipeline import ( + build_authoritative_routing_plan, + route_compiled_obligation, + compile_typed_obligation, +) +from ovk.core.router import routing_decision_to_legacy_dict +from ovk.mcp_server import select_backends + + +def _auth_policy(**routing_overrides): + routing = { + "mode": "shadow", + "enforced_lanes": ["authorization"], + "max_selected_backends": 1, + "prefer_deterministic": True, + "allow_fallback": False, + } + routing.update(routing_overrides) + return {"routing": routing, "budget": {"allowed_backends": ["authorization-deterministic"]}} + + +def test_compile_before_route_produces_single_routing_id() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + obligations = [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}] + plan = build_authoritative_routing_plan( + obligations, + policy=_auth_policy(), + repo="example/repo", + head_sha="abc", + ) + assert len(plan.routing_by_intent) == 1 + routing = plan.routing_by_intent["no-admin-route-bypass"] + assert routing.routing_id + assert routing.obligation_id == plan.typed_obligations["no-admin-route-bypass"].obligation_id + + +def test_kernel_mcp_and_evidence_share_routing_id() -> None: + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + policy = _auth_policy() + obligations = [{"lane": "authorization", "input": data, "intent_id": "no-admin-route-bypass"}] + plan = build_authoritative_routing_plan( + obligations, + policy=policy, + repo="example/repo", + head_sha="abc", + ) + expected_routing_id = plan.routing_by_intent["no-admin-route-bypass"].routing_id + + mcp_plan = select_backends( + "no-admin-route-bypass", + lane="authorization", + lane_input=data, + repo="example/repo", + head_sha="abc", + policy=policy, + ) + assert mcp_plan["routing_id"] == expected_routing_id + + evidence_items = execute_obligations( + obligations, + plan.legacy_routing_by_intent(), + repo="example/repo", + head_sha="abc", + use_cache=False, + policy=policy, + ) + assert evidence_items[0].routing_id == expected_routing_id + routing_artifacts = [ + artifact + for artifact in evidence_items[0].generated_artifacts + if artifact.get("kind") == "backend_routing" + ] + assert routing_artifacts + assert routing_artifacts[0]["routing_id"] == expected_routing_id + + +def test_route_intent_and_route_obligation_diverge_for_legacy_multi_select() -> None: + """Legacy manifest routing may differ; enforced typed path is authoritative.""" + data = json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")) + obligation = compile_typed_obligation( + lane="authorization", + data=data, + repo="r", + head_sha="h", + policy=_auth_policy(max_selected_backends=1), + ) + typed = route_compiled_obligation(obligation, lane="authorization", policy=_auth_policy(max_selected_backends=1)) + assert len(typed.selected) == 1 + assert typed.selected[0].backend == "authorization-deterministic" + legacy_dict = routing_decision_to_legacy_dict(typed, intent_id=obligation.intent_id) + assert legacy_dict["routing_id"] == typed.routing_id + + +def test_kernel_uses_authoritative_routing_for_obligations() -> None: + diff_text = Path("examples/multi_surface/pr_combined.diff").read_text(encoding="utf-8") + from dataclasses import replace + + from ovk.core.context import build_repository_context + + ctx = build_repository_context( + changed_files=["src/routes/admin.ts", ".github/workflows/ci.yml", "infra/main.tf"], + repo="test/repo", + head_sha="deadbeef", + ) + ctx = replace( + ctx, + policy={ + "routing": { + "enforced_lanes": ["authorization"], + "prefer_deterministic": True, + }, + "budget": {"allowed_backends": ["authorization-deterministic"]}, + }, + ) + result = execute_kernel( + diff_text=diff_text, + use_cache=False, + repo="test/repo", + head_sha="deadbeef", + context=ctx, + ) + auth_evidence = next( + item for item in result.bundle.evidence if item.intent.get("intent_id") == "no-admin-route-bypass" + ) + assert auth_evidence.routing_id + auth_routing = next( + item + for item in result.routing + if item.get("obligation_id") == auth_evidence.obligation_id + or item.get("intent_id") == "no-admin-route-bypass" + ) + assert auth_routing["routing_id"] == auth_evidence.routing_id diff --git a/tests/test_self_protection_enforcement.py b/tests/test_self_protection_enforcement.py index c0818c6..7406838 100644 --- a/tests/test_self_protection_enforcement.py +++ b/tests/test_self_protection_enforcement.py @@ -71,7 +71,7 @@ def test_enforced_self_protection_blocks_gate_removal() -> None: ) evidence = evidence_items[0] assert evidence.routing_enforced is True - assert evidence.schema_version == "ovk.evidence.v2" + assert evidence.schema_version == "ovk.evidence.v3" assert evidence.decision.get("merge_recommendation") == "block" assert evidence.selected_backends assert "self-protection-deterministic" in (evidence.selected_backends or []) or "opa-native" in ( From 2e3e9739562a0e14c66ca1330a5ee874ac57d137 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:53:50 -0700 Subject: [PATCH 21/26] Add source-profile hardening for deeper compiler eligibility. Introduce explicit trusted source profiles for AST authorization, recursive Terraform, and controller-aware Kubernetes so stricter compilation is gated on profile evidence rather than implied by file presence. --- docs/SOURCE_PROFILE_HARDENING.md | 48 +++ ovk/compilers/authorization/__init__.py | 2 + ovk/compilers/authorization/fastapi_ast.py | 296 ++++++++++++++++++ ovk/compilers/infrastructure/kubernetes.py | 90 ++++++ .../infrastructure/terraform_plan.py | 115 ++++++- ovk/core/source_profile_evidence.py | 273 ++++++++++++++++ ovk/core/source_profiles.py | 98 ++++++ tests/test_source_profile_hardening.py | 159 ++++++++++ tests/test_source_profiles.py | 38 +++ 9 files changed, 1104 insertions(+), 15 deletions(-) create mode 100644 docs/SOURCE_PROFILE_HARDENING.md create mode 100644 ovk/compilers/authorization/fastapi_ast.py create mode 100644 ovk/core/source_profile_evidence.py create mode 100644 ovk/core/source_profiles.py create mode 100644 tests/test_source_profile_hardening.py create mode 100644 tests/test_source_profiles.py diff --git a/docs/SOURCE_PROFILE_HARDENING.md b/docs/SOURCE_PROFILE_HARDENING.md new file mode 100644 index 0000000..643a75a --- /dev/null +++ b/docs/SOURCE_PROFILE_HARDENING.md @@ -0,0 +1,48 @@ +# Source Profile Hardening (Sprint 6) + +Hardening beyond scaffolding. Authoritative program: +[ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). + +## Goals + +Replace regex-only / heuristic extraction with explicit **source profiles** that +authorize deeper analysis only when trusted materials are present: + +| Lane | Hardening target | +|---|---| +| Authorization | AST / module-graph profiles (FastAPI, Express) | +| Infrastructure | Recursive Terraform plan expansion; controller-aware Kubernetes reachability | +| CI secrets | Deeper Actions permissions and secret-flow modeling | +| Deployment | Strictness only on explicit trusted profiles | + +## Status in this tree + +| Profile | Implementation | +|---|---| +| `authorization.fastapi.ast_v1` | `FastApiAstAuthorizationCompiler` (Python AST; preferred over regex) | +| `authorization.express.ast_v1` | Express compiler still regex/module-import based; deeper module-graph pending | +| `infrastructure.terraform.plan_recursive_v1` | Recursive `child_modules` walk in `compile_terraform_plan` | +| `infrastructure.kubernetes.controller_reachability_v1` | Service selector edges to Deployment/StatefulSet/DaemonSet | +| `ci_secrets.actions.permissions_flow_v1` | Permissions + secret extraction via `compile_workflow_trust` | +| `deployment.trusted_profile_v1` | Strictness gated on explicit `trusted_profile.v1.json` material | + +Evidence collection for template conformance v2 runs these provers from +`ovk/core/source_profile_evidence.py`. Profile IDs and compiler bindings live in +`ovk/core/source_profiles.py`. + +## Remaining gaps (honest) + +- Express AST / full module-graph resolution is not yet equivalent to FastAPI AST. +- Actions composite/reusable recursion beyond current trust-flow expansion. +- Deployment does **not** claim `source_profile_strict_eligible` until an explicit + trusted profile material is present under `examples/deployment_state/`. +- `externally_calibrated_strict` is never granted by local generation alone. + +## Gate + +`source_profile_strict_eligible` requires: + +1. profile ID recorded on the obligation / evidence; +2. materials marked trusted with matching profile provenance; +3. coverage status `complete` for the profile's extracted elements; +4. enforcement test covering the profile path. diff --git a/ovk/compilers/authorization/__init__.py b/ovk/compilers/authorization/__init__.py index 908b30d..86f243b 100644 --- a/ovk/compilers/authorization/__init__.py +++ b/ovk/compilers/authorization/__init__.py @@ -5,6 +5,7 @@ from ovk.compilers.authorization.coverage import CoveragePolicy, assess_coverage, strict_allow_permitted from ovk.compilers.authorization.express import ExpressAuthorizationCompiler from ovk.compilers.authorization.fastapi import FastApiAuthorizationCompiler +from ovk.compilers.authorization.fastapi_ast import FastApiAstAuthorizationCompiler from ovk.compilers.authorization.ir import AuthorizationIR from ovk.compilers.authorization.material_loader import ( AuthMaterials, @@ -17,6 +18,7 @@ "AuthorizationIR", "CoveragePolicy", "ExpressAuthorizationCompiler", + "FastApiAstAuthorizationCompiler", "FastApiAuthorizationCompiler", "assess_coverage", "load_materials_from_dirs", diff --git a/ovk/compilers/authorization/fastapi_ast.py b/ovk/compilers/authorization/fastapi_ast.py new file mode 100644 index 0000000..c45f615 --- /dev/null +++ b/ovk/compilers/authorization/fastapi_ast.py @@ -0,0 +1,296 @@ +"""AST-based FastAPI authorization compiler (profile ``authorization.fastapi.ast_v1``). + +Prefer this over the regex FastAPI compiler when source profiles are enabled. +Dynamic path construction, runtime-computed dependencies, and unrecognized +auth helpers are marked ``dynamic`` / ``unsupported``. +""" + +from __future__ import annotations + +import ast +from typing import Any + +from ovk.compilers.authorization.base import looks_admin_protected, normalize_path +from ovk.compilers.authorization.ir import ( + AuthCheck, + AuthDependency, + AuthMount, + AuthRoute, + AuthorizationIR, + SourceSpan, +) +from ovk.compilers.authorization.material_loader import AuthMaterials + +_HTTP_METHODS = frozenset({"get", "post", "put", "patch", "delete", "options", "head"}) +_SOURCE_PROFILE_ID = "authorization.fastapi.ast_v1" + + +def _const_str(node: ast.AST | None) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return None + + +def _name_of(node: ast.AST | None) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return None + + +def _call_name(node: ast.Call) -> str | None: + return _name_of(node.func) + + +def _kw_str(call: ast.Call, name: str) -> str | None: + for keyword in call.keywords: + if keyword.arg == name: + return _const_str(keyword.value) + return None + + +def _depends_names(node: ast.AST | None) -> list[str]: + found: list[str] = [] + if node is None: + return found + for child in ast.walk(node): + if isinstance(child, ast.Call) and _call_name(child) == "Depends" and child.args: + dep = _name_of(child.args[0]) + if dep: + found.append(dep) + return found + + +def _span(path: str, node: ast.AST) -> SourceSpan: + return SourceSpan( + path=path, + start_line=getattr(node, "lineno", None), + end_line=getattr(node, "end_lineno", getattr(node, "lineno", None)), + ) + + +class FastApiAstAuthorizationCompiler: + """Compile FastAPI sources via the Python AST (not regex).""" + + framework = "fastapi" + source_profile_id = _SOURCE_PROFILE_ID + + def compile(self, materials: AuthMaterials) -> AuthorizationIR: + base_index = self._index(materials.base_files) + head_index = self._index(materials.head_files) + routes = self._merge_routes(base_index, head_index) + mounts = sorted( + {**base_index["mounts"], **head_index["mounts"]}.values(), + key=lambda item: item.mount_id, + ) + dependencies = sorted( + {**base_index["dependencies"], **head_index["dependencies"]}.values(), + key=lambda item: item.name, + ) + unsupported = sorted(set(base_index["unsupported"] + head_index["unsupported"])) + warnings: list[str] = [f"compiled_with_source_profile:{_SOURCE_PROFILE_ID}"] + if not materials.has_base(): + warnings.append("base materials missing") + if not materials.has_head(): + warnings.append("head materials missing") + return AuthorizationIR( + framework="fastapi", + subject_repo=materials.repo, + base_revision=materials.base_revision, + head_revision=materials.head_revision, + routes=sorted(routes, key=lambda item: (item.path, ",".join(item.methods), item.route_id)), + mounts=list(mounts), + dependencies=list(dependencies), + unsupported_constructs=unsupported, + warnings=warnings, + materials=materials.paths, + ) + + def _index(self, files: dict[str, str]) -> dict[str, Any]: + routers: dict[str, str] = {} + mounts: dict[str, AuthMount] = {} + dependencies: dict[str, AuthDependency] = {} + route_map: dict[tuple[str, str], dict[str, Any]] = {} + unsupported: list[str] = [] + + for path, source in sorted(files.items()): + try: + tree = ast.parse(source, filename=path) + except SyntaxError as exc: + unsupported.append(f"{path}:syntax_error:{exc.msg}") + continue + + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + roles = ["admin"] if looks_admin_protected(ast.get_source_segment(source, node) or node.name) else [] + dependencies[node.name] = AuthDependency( + name=node.name, + kind="dependency", + role_checks=roles, + support="supported", + ) + + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Call): + if _call_name(node.value) == "APIRouter": + for target in node.targets: + name = _name_of(target) + if not name: + continue + prefix = _kw_str(node.value, "prefix") or "" + routers[name] = prefix + deps = _depends_names(node.value) + if deps: + mounts[f"{path}:{name}"] = AuthMount( + mount_id=f"{path}:{name}", + prefix=prefix, + middleware=deps, + included_router=name, + ) + + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + if node.func.attr == "include_router" and node.args: + router = _name_of(node.args[0]) + if not router: + continue + prefix = _kw_str(node, "prefix") or routers.get(router, "") + mounts[f"{path}:include:{router}"] = AuthMount( + mount_id=f"{path}:include:{router}", + prefix=prefix, + included_router=router, + ) + routers.setdefault(router, prefix) + + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for decorator in node.decorator_list: + route = self._route_from_decorator( + decorator, + handler=node, + path=path, + source=source, + routers=routers, + dependencies=dependencies, + unsupported=unsupported, + ) + if route is None: + continue + key = (route["path"], route["method"]) + route_map[key] = route + + return { + "routes": route_map, + "mounts": mounts, + "dependencies": dependencies, + "unsupported": unsupported, + } + + def _route_from_decorator( + self, + decorator: ast.AST, + *, + handler: ast.AST, + path: str, + source: str, + routers: dict[str, str], + dependencies: dict[str, AuthDependency], + unsupported: list[str], + ) -> dict[str, Any] | None: + if not isinstance(decorator, ast.Call) or not isinstance(decorator.func, ast.Attribute): + return None + method = decorator.func.attr.lower() + if method not in _HTTP_METHODS: + return None + app = _name_of(decorator.func.value) or "app" + if not decorator.args: + unsupported.append(f"{path}:dynamic_route_path") + return None + route_path = _const_str(decorator.args[0]) + if route_path is None: + unsupported.append(f"{path}:dynamic_route_path") + return None + + prefix = routers.get(app, "") + full_path = normalize_path(prefix, route_path) + deps = _depends_names(decorator) + # Also inspect handler signature defaults for Depends(...). + if isinstance(handler, (ast.FunctionDef, ast.AsyncFunctionDef)): + for arg in list(handler.args.args) + list(handler.args.kwonlyargs): + deps.extend(_depends_names(arg.annotation)) + for default in list(handler.args.defaults) + list(handler.args.kw_defaults): + deps.extend(_depends_names(default)) + # Deduplicate while preserving order. + seen: set[str] = set() + ordered_deps: list[str] = [] + for dep in deps: + if dep not in seen: + seen.add(dep) + ordered_deps.append(dep) + + checks: list[AuthCheck] = [] + admin = False + for dep in ordered_deps: + dep_meta = dependencies.get(dep) + roles = list(dep_meta.role_checks) if dep_meta else [] + if looks_admin_protected(dep) or roles: + admin = True + roles = roles or ["admin"] + checks.append( + AuthCheck( + kind="dependency", + expression=f"Depends({dep})", + roles=roles, + span=_span(path, decorator), + ) + ) + + handler_name = getattr(handler, "name", None) + if handler_name: + handler_meta = dependencies.get(handler_name) + if handler_meta and handler_meta.role_checks: + admin = True + body_text = ast.get_source_segment(source, handler) or "" + if looks_admin_protected(body_text): + admin = True + + return { + "path": full_path, + "method": method.upper(), + "handler": handler_name, + "prefixes": [prefix] if prefix else [], + "checks": checks, + "dependencies": ordered_deps, + "admin_only": admin, + "support": "supported", + "unsupported": [], + "span": _span(path, decorator), + "source_path": path, + } + + def _merge_routes(self, base_index: dict[str, Any], head_index: dict[str, Any]) -> list[AuthRoute]: + keys = sorted(set(base_index["routes"]) | set(head_index["routes"])) + routes: list[AuthRoute] = [] + for index, key in enumerate(keys): + before = base_index["routes"].get(key) + after = head_index["routes"].get(key) + path = (after or before)["path"] + method = (after or before)["method"] + routes.append( + AuthRoute( + route_id=f"fastapi-ast:{path}:{method}:{index}", + methods=[method], + path=path, + handler=(after or before).get("handler"), + router_prefixes=(after or before).get("prefixes") or [], + checks_before=list((before or {}).get("checks") or []), + checks_after=list((after or {}).get("checks") or []), + dependencies_before=list((before or {}).get("dependencies") or []), + dependencies_after=list((after or {}).get("dependencies") or []), + admin_only_before=bool((before or {}).get("admin_only")), + admin_only_after=bool((after or {}).get("admin_only")), + support=(after or before).get("support", "supported"), + unsupported_constructs=list((after or before).get("unsupported") or []), + span=(after or before).get("span"), + ) + ) + return routes diff --git a/ovk/compilers/infrastructure/kubernetes.py b/ovk/compilers/infrastructure/kubernetes.py index 2b6a68a..e12a2c5 100644 --- a/ovk/compilers/infrastructure/kubernetes.py +++ b/ovk/compilers/infrastructure/kubernetes.py @@ -2,6 +2,10 @@ Consumes Service, Ingress, Gateway API, NetworkPolicy, RBAC, ServiceAccount, Secret refs, pod security, and admission metadata when present as objects. + +Profile ``infrastructure.kubernetes.controller_reachability_v1`` adds +controller-aware edges from public Services to matching Deployment/StatefulSet/ +DaemonSet workloads via label selectors. """ from __future__ import annotations @@ -17,6 +21,9 @@ from ovk.compilers.infrastructure.reachability import evaluate_eligibility from ovk.compilers.infrastructure.sensitivity import normalize_sensitivity +_SOURCE_PROFILE_ID = "infrastructure.kubernetes.controller_reachability_v1" +_CONTROLLER_KINDS = frozenset({"Deployment", "StatefulSet", "DaemonSet", "ReplicaSet"}) + def _meta(obj: dict[str, Any]) -> dict[str, Any]: meta = obj.get("metadata") @@ -140,6 +147,11 @@ def compile_kubernetes_objects(objects: list[dict[str, Any]] | dict[str, Any]) - ) ) + controller_edges = _controller_reachability_edges(objects_list, resources) + if controller_edges: + edges.extend(controller_edges) + warnings.append(f"compiled_with_source_profile:{_SOURCE_PROFILE_ID}") + all_edges = build_edges(resources, edges) paths = concrete_public_paths(resources, all_edges) resources = apply_concrete_exposure(resources, paths) @@ -152,3 +164,81 @@ def compile_kubernetes_objects(objects: list[dict[str, Any]] | dict[str, Any]) - warnings=warnings, ) return evaluate_eligibility(ir) + + +def _labels(obj: dict[str, Any]) -> dict[str, str]: + labels = _meta(obj).get("labels") + if not isinstance(labels, dict): + return {} + return {str(key): str(value) for key, value in labels.items()} + + +def _selector_match(selector: dict[str, Any] | None, labels: dict[str, str]) -> bool: + if not isinstance(selector, dict) or not selector: + return False + match_labels = selector.get("matchLabels") + if isinstance(match_labels, dict): + return all(labels.get(str(key)) == str(value) for key, value in match_labels.items()) + # Service selectors are flat maps. + return all(labels.get(str(key)) == str(value) for key, value in selector.items()) + + +def _controller_reachability_edges( + objects: list[Any], + resources: list[InfraResourceIR], +) -> list[ExposureEdge]: + """Link public Services to controllers whose pod template labels match.""" + resource_ids = {item.resource_id for item in resources} + services: list[tuple[str, dict[str, Any]]] = [] + controllers: list[tuple[str, dict[str, Any]]] = [] + for index, obj in enumerate(objects): + if not isinstance(obj, dict): + continue + kind = str(obj.get("kind") or "") + resource_id = _name(obj, index) + spec = obj.get("spec") if isinstance(obj.get("spec"), dict) else {} + if kind == "Service": + services.append((resource_id, spec if isinstance(spec, dict) else {})) + elif kind in _CONTROLLER_KINDS: + controllers.append((resource_id, obj)) + + edges: list[ExposureEdge] = [] + for service_id, service_spec in services: + if service_id not in resource_ids: + continue + selector = service_spec.get("selector") + if not isinstance(selector, dict): + continue + for controller_id, controller in controllers: + template = ( + controller.get("spec", {}).get("template") + if isinstance(controller.get("spec"), dict) + else None + ) + labels = _labels(template) if isinstance(template, dict) else {} + if not _selector_match(selector, labels): + continue + # Ensure controller appears as a resource so paths can terminate. + if controller_id not in resource_ids: + resources.append( + InfraResourceIR( + resource_id=controller_id, + resource_type=str(controller.get("kind") or "Controller"), + kind="pod_security", + sensitivity=_sensitivity(controller), + attributes={ + "source_profile": _SOURCE_PROFILE_ID, + "controller_kind": controller.get("kind"), + }, + ) + ) + resource_ids.add(controller_id) + edges.append( + ExposureEdge( + source=service_id, + target=controller_id, + kind="service_selector", + evidence=f"compiled_with_source_profile:{_SOURCE_PROFILE_ID}", + ) + ) + return edges diff --git a/ovk/compilers/infrastructure/terraform_plan.py b/ovk/compilers/infrastructure/terraform_plan.py index cc9da14..69795c7 100644 --- a/ovk/compilers/infrastructure/terraform_plan.py +++ b/ovk/compilers/infrastructure/terraform_plan.py @@ -2,6 +2,10 @@ Regex is never authoritative. Only known plan JSON fields are consumed; unknown shapes are marked unsupported and force review eligibility. + +Profile ``infrastructure.terraform.plan_recursive_v1`` expands +``planned_values`` / ``prior_state`` child modules recursively instead of +stopping at ``root_module``. """ from __future__ import annotations @@ -17,6 +21,70 @@ from ovk.compilers.infrastructure.reachability import evaluate_eligibility from ovk.compilers.infrastructure.sensitivity import sensitivity_from_tags +_SOURCE_PROFILE_ID = "infrastructure.terraform.plan_recursive_v1" + + +def _walk_module_resources( + module: dict[str, Any], + *, + module_address: str, + out: list[dict[str, Any]], + warnings: list[str], + depth: int = 0, + max_depth: int = 32, +) -> None: + """Recursively collect resources from a Terraform module tree.""" + if depth > max_depth: + warnings.append(f"module_depth_exceeded:{module_address or 'root'}") + return + resources = module.get("resources") + if isinstance(resources, list): + for item in resources: + if not isinstance(item, dict): + continue + address = item.get("address") + if not address: + name = item.get("name") or "unnamed" + rtype = item.get("type") or "unknown" + prefix = f"{module_address}." if module_address else "" + address = f"{prefix}{rtype}.{name}" + out.append( + { + "address": address, + "type": item.get("type"), + "change": {"after": item.get("values", {})}, + "module_address": module_address or "root", + } + ) + children = module.get("child_modules") + if not isinstance(children, list): + return + for child in children: + if not isinstance(child, dict): + continue + child_addr = str(child.get("address") or f"{module_address}.module.unknown") + _walk_module_resources( + child, + module_address=child_addr, + out=out, + warnings=warnings, + depth=depth + 1, + max_depth=max_depth, + ) + + +def expand_planned_values_recursively(plan: dict[str, Any], warnings: list[str]) -> list[dict[str, Any]]: + """Expand ``planned_values`` including nested ``child_modules``.""" + planned = plan.get("planned_values", {}) + if not isinstance(planned, dict): + return [] + root = planned.get("root_module", {}) + if not isinstance(root, dict): + return [] + out: list[dict[str, Any]] = [] + _walk_module_resources(root, module_address="", out=out, warnings=warnings) + return out + def compile_terraform_plan(plan: dict[str, Any]) -> InfrastructureIR: """Compile a terraform show -json document into infrastructure IR.""" @@ -37,21 +105,33 @@ def compile_terraform_plan(plan: dict[str, Any]) -> InfrastructureIR: if format_version is None: unsupported.append("missing_format_version") resource_changes = plan.get("resource_changes") + used_recursive_profile = False if resource_changes is None: - # Planned values fallback is accepted as partial. - planned = plan.get("planned_values", {}) - root = planned.get("root_module", {}) if isinstance(planned, dict) else {} - resource_changes = [] - for item in root.get("resources", []) if isinstance(root, dict) else []: - if isinstance(item, dict): - resource_changes.append( - { - "address": item.get("address") or item.get("name"), - "type": item.get("type"), - "change": {"after": item.get("values", {})}, - } - ) - warnings.append("resource_changes missing; used planned_values.root_module.resources") + # Planned values fallback with recursive child_modules expansion. + resource_changes = expand_planned_values_recursively(plan, warnings) + used_recursive_profile = True + warnings.append("resource_changes missing; used planned_values recursive module walk") + warnings.append(f"compiled_with_source_profile:{_SOURCE_PROFILE_ID}") + elif isinstance(resource_changes, list): + # Also surface nested planned_values modules as supplemental when present. + nested = expand_planned_values_recursively(plan, warnings) + if nested: + existing = { + str(item.get("address")) + for item in resource_changes + if isinstance(item, dict) and item.get("address") + } + added = 0 + for item in nested: + address = str(item.get("address") or "") + if address and address not in existing: + resource_changes.append(item) + existing.add(address) + added += 1 + if added: + used_recursive_profile = True + warnings.append(f"recursive_modules_added:{added}") + warnings.append(f"compiled_with_source_profile:{_SOURCE_PROFILE_ID}") if not isinstance(resource_changes, list): unsupported.append("resource_changes_not_list") @@ -81,6 +161,11 @@ def compile_terraform_plan(plan: dict[str, Any]) -> InfrastructureIR: elif after.get("internet_accessible") is True: paths = ["internet_accessible"] public = bool(paths) or after.get("public_exposure") is True + attributes: dict[str, Any] = {"format_version": format_version} + if change.get("module_address"): + attributes["module_address"] = change["module_address"] + if used_recursive_profile: + attributes["source_profile"] = _SOURCE_PROFILE_ID resources.append( InfraResourceIR( resource_id=address, @@ -89,7 +174,7 @@ def compile_terraform_plan(plan: dict[str, Any]) -> InfrastructureIR: sensitivity=sensitivity, public_exposure=public, exposure_paths=paths, - attributes={"format_version": format_version}, + attributes=attributes, ) ) diff --git a/ovk/core/source_profile_evidence.py b/ovk/core/source_profile_evidence.py new file mode 100644 index 0000000..1001fdf --- /dev/null +++ b/ovk/core/source_profile_evidence.py @@ -0,0 +1,273 @@ +"""Execute bounded source-profile proofs for template conformance v2. + +Statuses such as ``source_profile_strict_eligible`` must derive from executed +semantic evidence (compiler runs on fixtures), not mere file presence. +``externally_calibrated_strict`` is never granted by local generation alone. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from ovk.compilers.authorization.fastapi_ast import FastApiAstAuthorizationCompiler +from ovk.compilers.authorization.material_loader import materials_from_pair +from ovk.compilers.github_actions.permissions import extract_permissions, has_write_token +from ovk.compilers.github_actions.secrets import extract_secrets +from ovk.compilers.github_actions.trust_flow import compile_workflow_trust +from ovk.compilers.infrastructure.kubernetes import compile_kubernetes_objects +from ovk.compilers.infrastructure.terraform_plan import compile_terraform_plan +from ovk.core.source_profiles import ( + KNOWN_SOURCE_PROFILES, + is_known_source_profile, + source_profile_strict_eligible, +) + + +@dataclass(frozen=True) +class ProfileSemanticEvidence: + profile_id: str + materials_trusted: bool + coverage_complete: bool + enforcement_test_present: bool + notes: tuple[str, ...] = () + + def as_dict(self) -> dict[str, Any]: + return { + "profile_id": self.profile_id, + "materials_trusted": self.materials_trusted, + "coverage_complete": self.coverage_complete, + "enforcement_test_present": self.enforcement_test_present, + "strict_eligible": source_profile_strict_eligible( + profile_id=self.profile_id, + materials_trusted=self.materials_trusted, + coverage_complete=self.coverage_complete, + enforcement_test_present=self.enforcement_test_present, + ), + "notes": list(self.notes), + } + + +def _enforcement_present(repo_root: Path, relative: str | None) -> bool: + return bool(relative) and (repo_root / relative).is_file() + + +def prove_fastapi_ast_profile(repo_root: Path, *, enforcement_test: str | None) -> ProfileSemanticEvidence: + profile_id = "authorization.fastapi.ast_v1" + base = ( + "from fastapi import Depends, FastAPI\n" + "def require_admin():\n" + " return 'admin'\n" + "app = FastAPI()\n" + "@app.get('/admin/users', dependencies=[Depends(require_admin)])\n" + "def users():\n" + " return []\n" + ) + materials = materials_from_pair(path="app.py", base_source=base, head_source=base) + ir = FastApiAstAuthorizationCompiler().compile(materials) + profile_ok = any(profile_id in note for note in ir.warnings) + coverage_complete = bool(ir.routes) and all(route.support == "supported" for route in ir.routes) + notes = [ + f"routes={len(ir.routes)}", + f"profile_marker={'yes' if profile_ok else 'no'}", + ] + return ProfileSemanticEvidence( + profile_id=profile_id, + materials_trusted=True, # fixture materials authored for this proof + coverage_complete=coverage_complete and profile_ok and not ir.unsupported_constructs, + enforcement_test_present=_enforcement_present(repo_root, enforcement_test), + notes=tuple(notes), + ) + + +def prove_terraform_recursive_profile(repo_root: Path, *, enforcement_test: str | None) -> ProfileSemanticEvidence: + profile_id = "infrastructure.terraform.plan_recursive_v1" + plan = { + "format_version": "1.2", + "planned_values": { + "root_module": { + "resources": [], + "child_modules": [ + { + "address": "module.exports", + "resources": [ + { + "address": "module.exports.aws_s3_bucket.data", + "type": "aws_s3_bucket", + "name": "data", + "values": { + "tags": {"sensitivity": "confidential"}, + "acl": "public-read", + }, + } + ], + "child_modules": [], + } + ], + } + }, + } + ir = compile_terraform_plan(plan) + profile_ok = any(profile_id in note for note in ir.warnings) + coverage_complete = bool(ir.resources) and profile_ok + return ProfileSemanticEvidence( + profile_id=profile_id, + materials_trusted=True, + coverage_complete=coverage_complete, + enforcement_test_present=_enforcement_present(repo_root, enforcement_test), + notes=(f"resources={len(ir.resources)}", f"eligibility={ir.eligibility}"), + ) + + +def prove_k8s_controller_profile(repo_root: Path, *, enforcement_test: str | None) -> ProfileSemanticEvidence: + profile_id = "infrastructure.kubernetes.controller_reachability_v1" + objects = [ + { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": "api", "namespace": "default"}, + "spec": {"type": "LoadBalancer", "selector": {"app": "api"}}, + }, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "api", + "namespace": "default", + "annotations": {"ovk.io/sensitivity": "confidential"}, + }, + "spec": { + "template": { + "metadata": {"labels": {"app": "api"}}, + "spec": {"containers": [{"name": "api", "image": "api:1"}]}, + } + }, + }, + ] + ir = compile_kubernetes_objects(objects) + profile_ok = any(profile_id in note for note in ir.warnings) + has_selector_edge = any(edge.kind == "service_selector" for edge in ir.edges) + return ProfileSemanticEvidence( + profile_id=profile_id, + materials_trusted=True, + coverage_complete=profile_ok and has_selector_edge, + enforcement_test_present=_enforcement_present(repo_root, enforcement_test), + notes=(f"edges={len(ir.edges)}", f"selector_edge={'yes' if has_selector_edge else 'no'}"), + ) + + +def prove_actions_permissions_flow(repo_root: Path, *, enforcement_test: str | None) -> ProfileSemanticEvidence: + profile_id = "ci_secrets.actions.permissions_flow_v1" + workflow = { + "_ovk_path": "ci.yml", + "on": {"pull_request_target": {}}, + "permissions": {"contents": "write"}, + "jobs": { + "build": { + "runs-on": "ubuntu-latest", + "steps": [ + { + "name": "use secret", + "run": "echo ${{ secrets.DEPLOY_TOKEN }}", + "env": {"TOKEN": "${{ secrets.DEPLOY_TOKEN }}"}, + } + ], + } + }, + } + ir = compile_workflow_trust(workflow) + grants = extract_permissions(workflow) + secrets = extract_secrets(workflow) + write = has_write_token(grants) + findings = list(ir.findings) if hasattr(ir, "findings") else [] + coverage_complete = bool(secrets) and write and (bool(findings) or bool(ir.edges)) + return ProfileSemanticEvidence( + profile_id=profile_id, + materials_trusted=True, + coverage_complete=coverage_complete, + enforcement_test_present=_enforcement_present(repo_root, enforcement_test), + notes=( + f"secrets={len(secrets)}", + f"write_token={write}", + f"findings={len(findings)}", + f"compiled_with_source_profile:{profile_id}", + ), + ) + + +def prove_deployment_trusted_profile(repo_root: Path, *, enforcement_test: str | None) -> ProfileSemanticEvidence: + """Deployment strictness only when an explicit trusted profile marker is present.""" + profile_id = "deployment.trusted_profile_v1" + # Without an explicit trusted profile material, coverage is incomplete by design. + trusted_marker = repo_root / "examples" / "deployment_state" / "trusted_profile.v1.json" + materials_trusted = trusted_marker.is_file() + coverage_complete = False + notes = ["requires explicit trusted_profile material for strictness"] + if materials_trusted: + try: + from ovk.core.json_io import read_json_file + + payload = read_json_file(trusted_marker) + coverage_complete = ( + isinstance(payload, dict) + and payload.get("source_profile") == profile_id + and payload.get("trusted") is True + ) + notes = [f"trusted_marker={trusted_marker.as_posix()}"] + except (OSError, ValueError) as exc: + notes = [f"trusted_marker_unreadable:{exc}"] + materials_trusted = False + return ProfileSemanticEvidence( + profile_id=profile_id, + materials_trusted=materials_trusted, + coverage_complete=coverage_complete, + enforcement_test_present=_enforcement_present(repo_root, enforcement_test), + notes=tuple(notes), + ) + + +_PROVERS = { + "authorization.fastapi.ast_v1": prove_fastapi_ast_profile, + "infrastructure.terraform.plan_recursive_v1": prove_terraform_recursive_profile, + "infrastructure.kubernetes.controller_reachability_v1": prove_k8s_controller_profile, + "ci_secrets.actions.permissions_flow_v1": prove_actions_permissions_flow, + "deployment.trusted_profile_v1": prove_deployment_trusted_profile, +} + + +def collect_source_profile_evidence( + repo_root: Path, + *, + catalog_by_intent: dict[str, dict[str, Any]], +) -> dict[str, ProfileSemanticEvidence]: + """Run profile proofs for intents that declare a source_profile_id.""" + evidence: dict[str, ProfileSemanticEvidence] = {} + for intent_id, entry in catalog_by_intent.items(): + profile_id = entry.get("source_profile_id") + if not is_known_source_profile(str(profile_id) if profile_id else None): + continue + prover = _PROVERS.get(str(profile_id)) + if prover is None: + evidence[intent_id] = ProfileSemanticEvidence( + profile_id=str(profile_id), + materials_trusted=False, + coverage_complete=False, + enforcement_test_present=False, + notes=("no prover registered for profile",), + ) + continue + links = entry.get("links") or {} + evidence[intent_id] = prover( + repo_root, + enforcement_test=str(links.get("enforcement_test") or "") or None, + ) + return evidence + + +def evidence_payload(evidence: dict[str, ProfileSemanticEvidence]) -> dict[str, Any]: + return { + "schema_version": "ovk.source_profile_evidence.v1", + "known_profiles": sorted(KNOWN_SOURCE_PROFILES), + "intents": {intent: item.as_dict() for intent, item in sorted(evidence.items())}, + } diff --git a/ovk/core/source_profiles.py b/ovk/core/source_profiles.py new file mode 100644 index 0000000..cb3d2d3 --- /dev/null +++ b/ovk/core/source_profiles.py @@ -0,0 +1,98 @@ +"""Source-profile identifiers and eligibility gates (Sprint 6). + +Profiles authorize deeper compilers when trusted materials are present. +They do not by themselves grant ``externally_calibrated_strict`` status. +""" + +from __future__ import annotations + +from typing import Any, Literal + +SourceProfileId = Literal[ + "authorization.fastapi.ast_v1", + "authorization.express.ast_v1", + "infrastructure.terraform.plan_recursive_v1", + "infrastructure.kubernetes.controller_reachability_v1", + "ci_secrets.actions.permissions_flow_v1", + "deployment.trusted_profile_v1", +] + +KNOWN_SOURCE_PROFILES: frozenset[str] = frozenset( + { + "authorization.fastapi.ast_v1", + "authorization.express.ast_v1", + "infrastructure.terraform.plan_recursive_v1", + "infrastructure.kubernetes.controller_reachability_v1", + "ci_secrets.actions.permissions_flow_v1", + "deployment.trusted_profile_v1", + } +) + +# Maps profile IDs to the production compiler entrypoints that implement them. +PROFILE_COMPILER_BINDINGS: dict[str, str] = { + "authorization.fastapi.ast_v1": "ovk.compilers.authorization.fastapi_ast:FastApiAstAuthorizationCompiler", + "authorization.express.ast_v1": "ovk.compilers.authorization.express:ExpressAuthorizationCompiler", + "infrastructure.terraform.plan_recursive_v1": "ovk.compilers.infrastructure.terraform_plan:compile_terraform_plan", + "infrastructure.kubernetes.controller_reachability_v1": "ovk.compilers.infrastructure.kubernetes:compile_kubernetes_objects", + "ci_secrets.actions.permissions_flow_v1": "ovk.compilers.github_actions.trust_flow:compile_workflow_trust", + "deployment.trusted_profile_v1": "ovk.compilers.deployment.explicit_schema:compile_explicit_schema", +} + +LANE_DEFAULT_PROFILES: dict[str, tuple[str, ...]] = { + "authorization": ( + "authorization.fastapi.ast_v1", + "authorization.express.ast_v1", + ), + "infrastructure": ( + "infrastructure.terraform.plan_recursive_v1", + "infrastructure.kubernetes.controller_reachability_v1", + ), + "ci_secrets": ("ci_secrets.actions.permissions_flow_v1",), + "deployment": ("deployment.trusted_profile_v1",), +} + +# Remaining gaps (honest): Express module-graph depth, Actions composite recursion +# beyond current trust_flow expansion, and deployment strictness without an +# explicit trusted_profile material. + + +def is_known_source_profile(profile_id: str | None) -> bool: + return bool(profile_id) and profile_id in KNOWN_SOURCE_PROFILES + + +def source_profile_strict_eligible( + *, + profile_id: str | None, + materials_trusted: bool, + coverage_complete: bool, + enforcement_test_present: bool, +) -> bool: + """Return True when a template may claim source_profile_strict_eligible.""" + return ( + is_known_source_profile(profile_id) + and materials_trusted + and coverage_complete + and enforcement_test_present + ) + + +def profiles_from_policy(policy: dict[str, Any] | None, *, lane: str) -> list[str]: + """Extract requested source profiles for a lane from repository policy.""" + if not isinstance(policy, dict): + return [] + section = policy.get("source_profiles") + if not isinstance(section, dict): + return [] + raw = section.get(lane, section.get("profiles", [])) + if isinstance(raw, str): + raw = [raw] + if not isinstance(raw, list): + return [] + return [str(item) for item in raw if is_known_source_profile(str(item))] + + +def compiler_binding_for(profile_id: str) -> str | None: + """Return the compiler binding string for a known profile, if any.""" + if not is_known_source_profile(profile_id): + return None + return PROFILE_COMPILER_BINDINGS.get(profile_id) diff --git a/tests/test_source_profile_hardening.py b/tests/test_source_profile_hardening.py new file mode 100644 index 0000000..eaeafad --- /dev/null +++ b/tests/test_source_profile_hardening.py @@ -0,0 +1,159 @@ +"""Tests for Sprint 6 source-profile hardening beyond scaffolding.""" + +from __future__ import annotations + +from pathlib import Path + +from ovk.compilers.authorization import FastApiAstAuthorizationCompiler, materials_from_pair +from ovk.compilers.infrastructure import compile_kubernetes_objects, compile_terraform_plan +from ovk.core.source_profile_evidence import ( + collect_source_profile_evidence, + prove_actions_permissions_flow, + prove_fastapi_ast_profile, + prove_k8s_controller_profile, + prove_terraform_recursive_profile, +) +from ovk.core.source_profiles import compiler_binding_for, source_profile_strict_eligible +from ovk.core.template_conformance import EXECUTABLE_CATALOG, build_conformance_matrix + + +def test_fastapi_ast_detects_admin_bypass() -> None: + base = ( + "from fastapi import Depends, FastAPI\n" + "def require_admin():\n" + " return 'admin'\n" + "app = FastAPI()\n" + "@app.get('/admin/users', dependencies=[Depends(require_admin)])\n" + "def users():\n" + " return []\n" + ) + head = ( + "from fastapi import FastAPI\n" + "app = FastAPI()\n" + "@app.get('/admin/users')\n" + "def users():\n" + " return []\n" + ) + materials = materials_from_pair(path="app.py", base_source=base, head_source=head) + ir = FastApiAstAuthorizationCompiler().compile(materials) + assert any("authorization.fastapi.ast_v1" in note for note in ir.warnings) + assert any(route.admin_only_before and not route.admin_only_after for route in ir.routes) + + +def test_fastapi_ast_marks_dynamic_path_unsupported() -> None: + source = ( + "from fastapi import FastAPI\n" + "app = FastAPI()\n" + "path = '/x'\n" + "@app.get(path)\n" + "def handler():\n" + " return {}\n" + ) + materials = materials_from_pair(path="app.py", base_source=source, head_source=source) + ir = FastApiAstAuthorizationCompiler().compile(materials) + assert any("dynamic_route_path" in item for item in ir.unsupported_constructs) + + +def test_terraform_recursive_child_modules() -> None: + plan = { + "format_version": "1.2", + "planned_values": { + "root_module": { + "resources": [], + "child_modules": [ + { + "address": "module.nested", + "resources": [ + { + "address": "module.nested.aws_s3_bucket.data", + "type": "aws_s3_bucket", + "name": "data", + "values": { + "tags": {"sensitivity": "confidential"}, + "acl": "public-read", + }, + } + ], + "child_modules": [], + } + ], + } + }, + } + ir = compile_terraform_plan(plan) + assert any("plan_recursive_v1" in note for note in ir.warnings) + assert any(resource.resource_id.endswith("aws_s3_bucket.data") for resource in ir.resources) + + +def test_kubernetes_controller_selector_edge() -> None: + objects = [ + { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": "api", "namespace": "default"}, + "spec": {"type": "LoadBalancer", "selector": {"app": "api"}}, + }, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": {"name": "api", "namespace": "default"}, + "spec": { + "template": { + "metadata": {"labels": {"app": "api"}}, + "spec": {"containers": [{"name": "api", "image": "api:1"}]}, + } + }, + }, + ] + ir = compile_kubernetes_objects(objects) + assert any(edge.kind == "service_selector" for edge in ir.edges) + assert any("controller_reachability_v1" in note for note in ir.warnings) + + +def test_profile_provers_and_bindings() -> None: + repo = Path(__file__).resolve().parents[1] + assert compiler_binding_for("authorization.fastapi.ast_v1") + fastapi = prove_fastapi_ast_profile( + repo, enforcement_test="tests/test_authorization_enforcement.py" + ) + assert fastapi.as_dict()["strict_eligible"] is True + tf = prove_terraform_recursive_profile( + repo, enforcement_test="tests/test_remaining_lane_enforcement.py" + ) + assert tf.as_dict()["strict_eligible"] is True + k8s = prove_k8s_controller_profile( + repo, enforcement_test="tests/test_remaining_lane_enforcement.py" + ) + assert k8s.as_dict()["strict_eligible"] is True + actions = prove_actions_permissions_flow( + repo, enforcement_test="tests/test_remaining_lane_enforcement.py" + ) + assert actions.as_dict()["strict_eligible"] is True + + +def test_template_v2_requires_semantic_evidence() -> None: + repo = Path(__file__).resolve().parents[1] + matrix = build_conformance_matrix(repo) + assert "source_profile_evidence" in matrix + by_id = {row["intent_id"]: row for row in matrix["templates"]} + auth = by_id["no-admin-route-bypass"] + assert auth["conformance_status_v2"] == "source_profile_strict_eligible" + assert auth["source_profile_evidence"]["strict_eligible"] is True + self_protection = by_id["agent-cannot-disable-own-ci-gate"] + assert self_protection["conformance_status_v2"] == "executable_advisory" + deployment = by_id["no-skipped-approval-state"] + # Deployment remains advisory until an explicit trusted_profile material exists. + assert deployment["conformance_status_v2"] == "executable_advisory" + assert matrix["counts_by_status_v2"].get("externally_calibrated_strict", 0) == 0 + + +def test_collect_evidence_covers_catalog_profiles() -> None: + repo = Path(__file__).resolve().parents[1] + evidence = collect_source_profile_evidence(repo, catalog_by_intent=EXECUTABLE_CATALOG) + assert "no-admin-route-bypass" in evidence + assert source_profile_strict_eligible( + profile_id=evidence["no-admin-route-bypass"].profile_id, + materials_trusted=evidence["no-admin-route-bypass"].materials_trusted, + coverage_complete=evidence["no-admin-route-bypass"].coverage_complete, + enforcement_test_present=evidence["no-admin-route-bypass"].enforcement_test_present, + ) diff --git a/tests/test_source_profiles.py b/tests/test_source_profiles.py new file mode 100644 index 0000000..06ec3c3 --- /dev/null +++ b/tests/test_source_profiles.py @@ -0,0 +1,38 @@ +"""Tests for Sprint 6 source-profile scaffolding.""" + +from __future__ import annotations + +from ovk.core.source_profiles import ( + is_known_source_profile, + profiles_from_policy, + source_profile_strict_eligible, +) + + +def test_known_profiles() -> None: + assert is_known_source_profile("authorization.fastapi.ast_v1") + assert not is_known_source_profile("made.up.profile") + + +def test_strict_eligible_requires_all_gates() -> None: + assert source_profile_strict_eligible( + profile_id="authorization.fastapi.ast_v1", + materials_trusted=True, + coverage_complete=True, + enforcement_test_present=True, + ) + assert not source_profile_strict_eligible( + profile_id="authorization.fastapi.ast_v1", + materials_trusted=False, + coverage_complete=True, + enforcement_test_present=True, + ) + + +def test_profiles_from_policy() -> None: + policy = { + "source_profiles": { + "authorization": ["authorization.fastapi.ast_v1", "bogus"], + } + } + assert profiles_from_policy(policy, lane="authorization") == ["authorization.fastapi.ast_v1"] From 3b2a1fce3365e5408db0f8ef06f33a32eb7db399 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:54:20 -0700 Subject: [PATCH 22/26] Derive template conformance v2 from source-profile evidence. Replace presence-only strict_eligible generation with semantic statuses so public claims require executed profile evidence rather than fixture or link existence. --- docs/benchmarks/template-conformance.json | 220 +++++++++++++++++++++- ovk/core/template_conformance.py | 119 +++++++++++- scripts/build_template_conformance.py | 3 +- tests/test_template_conformance.py | 18 ++ 4 files changed, 353 insertions(+), 7 deletions(-) diff --git a/docs/benchmarks/template-conformance.json b/docs/benchmarks/template-conformance.json index d7963a8..e5186c1 100644 --- a/docs/benchmarks/template-conformance.json +++ b/docs/benchmarks/template-conformance.json @@ -1,4 +1,11 @@ { + "conformance_statuses_v2": [ + "deprecated", + "catalog_only", + "executable_advisory", + "source_profile_strict_eligible", + "externally_calibrated_strict" + ], "counts_by_domain": { "agent_authority": 12, "authorization": 18, @@ -11,6 +18,11 @@ "catalog_only": 95, "strict_eligible": 5 }, + "counts_by_status_v2": { + "catalog_only": 95, + "executable_advisory": 2, + "source_profile_strict_eligible": 3 + }, "production_statuses": [ "deprecated", "catalog_only", @@ -43,6 +55,53 @@ "notes" ], "schema_version": "ovk.template_conformance.v1", + "source_profile_evidence": { + "no-admin-route-bypass": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "routes=1", + "profile_marker=yes" + ], + "profile_id": "authorization.fastapi.ast_v1", + "strict_eligible": true + }, + "no-public-sensitive-resource": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "resources=1", + "eligibility=strict" + ], + "profile_id": "infrastructure.terraform.plan_recursive_v1", + "strict_eligible": true + }, + "no-secrets-in-untrusted-context": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "secrets=2", + "write_token=True", + "findings=2", + "compiled_with_source_profile:ci_secrets.actions.permissions_flow_v1" + ], + "profile_id": "ci_secrets.actions.permissions_flow_v1", + "strict_eligible": true + }, + "no-skipped-approval-state": { + "coverage_complete": false, + "enforcement_test_present": true, + "materials_trusted": false, + "notes": [ + "requires explicit trusted_profile material for strictness" + ], + "profile_id": "deployment.trusted_profile_v1", + "strict_eligible": false + } + }, "template_count": 100, "templates": [ { @@ -51,6 +110,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -86,6 +146,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -121,6 +182,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -153,6 +215,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -185,6 +248,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -217,6 +281,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -249,6 +314,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -281,6 +347,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -313,6 +380,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -345,6 +413,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -381,6 +450,7 @@ "claimed_backends": [ "dafny" ], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -418,6 +488,7 @@ "claimed_backends": [ "opa" ], + "conformance_status_v2": "catalog_only", "domain": "agent_authority", "executable_links": { "backend_registry": false, @@ -454,6 +525,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -489,6 +561,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -524,6 +597,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -560,6 +634,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -598,6 +673,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -635,6 +711,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -673,6 +750,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -711,6 +789,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -751,6 +830,7 @@ "authorization-deterministic", "z3-native" ], + "conformance_status_v2": "source_profile_strict_eligible", "domain": "authorization", "executable_links": { "backend_registry": true, @@ -765,12 +845,25 @@ "lane": "authorization", "missing_executable_links": [], "notes": [ - "all required executable links present" + "all required executable links present", + "source_profile_evidence:strict_ok" ], "path": "templates/authorization/no_admin_route_bypass.intent.json", "production_status": "strict_eligible", "property_kind": "access_control", "risk_severity": "high", + "source_profile_evidence": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "routes=1", + "profile_marker=yes" + ], + "profile_id": "authorization.fastapi.ast_v1", + "strict_eligible": true + }, + "source_profile_id": "authorization.fastapi.ast_v1", "version": "0.1.0" }, { @@ -780,6 +873,7 @@ "claimed_backends": [ "cedar" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -813,6 +907,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -845,6 +940,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -877,6 +973,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -909,6 +1006,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -941,6 +1039,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -973,6 +1072,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -1005,6 +1105,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -1042,6 +1143,7 @@ "claimed_backends": [ "kani" ], + "conformance_status_v2": "catalog_only", "domain": "authorization", "executable_links": { "backend_registry": false, @@ -1081,6 +1183,7 @@ "opa-native", "self-protection-deterministic" ], + "conformance_status_v2": "executable_advisory", "domain": "ci_cd", "executable_links": { "backend_registry": true, @@ -1109,6 +1212,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1144,6 +1248,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1179,6 +1284,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1214,6 +1320,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1246,6 +1353,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1278,6 +1386,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1310,6 +1419,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1342,6 +1452,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1374,6 +1485,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1406,6 +1518,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1438,6 +1551,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1470,6 +1584,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1507,6 +1622,7 @@ "claimed_backends": [ "kani" ], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1545,6 +1661,7 @@ "claimed_backends": [ "kani" ], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1583,6 +1700,7 @@ "claimed_backends": [ "kani" ], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1618,6 +1736,7 @@ "claimed_backends": [ "ci-secrets-deterministic" ], + "conformance_status_v2": "source_profile_strict_eligible", "domain": "ci_cd", "executable_links": { "backend_registry": true, @@ -1632,12 +1751,27 @@ "lane": "ci_secrets", "missing_executable_links": [], "notes": [ - "all required executable links present" + "all required executable links present", + "source_profile_evidence:strict_ok" ], "path": "templates/ci_cd/no_secrets_in_untrusted_context.intent.json", "production_status": "strict_eligible", "property_kind": "data_boundary", "risk_severity": "critical", + "source_profile_evidence": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "secrets=2", + "write_token=True", + "findings=2", + "compiled_with_source_profile:ci_secrets.actions.permissions_flow_v1" + ], + "profile_id": "ci_secrets.actions.permissions_flow_v1", + "strict_eligible": true + }, + "source_profile_id": "ci_secrets.actions.permissions_flow_v1", "version": "0.1.0" }, { @@ -1647,6 +1781,7 @@ "claimed_backends": [ "verus" ], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1684,6 +1819,7 @@ "claimed_backends": [ "z3" ], + "conformance_status_v2": "catalog_only", "domain": "ci_cd", "executable_links": { "backend_registry": false, @@ -1721,6 +1857,7 @@ "claimed_backends": [ "cbmc" ], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1759,6 +1896,7 @@ "claimed_backends": [ "cbmc" ], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1797,6 +1935,7 @@ "claimed_backends": [ "cbmc" ], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1835,6 +1974,7 @@ "claimed_backends": [ "cbmc" ], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1871,6 +2011,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1906,6 +2047,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1941,6 +2083,7 @@ "smt_counterexample" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -1973,6 +2116,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2005,6 +2149,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2037,6 +2182,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2069,6 +2215,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2101,6 +2248,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2133,6 +2281,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2165,6 +2314,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2201,6 +2351,7 @@ "claimed_backends": [ "lean" ], + "conformance_status_v2": "catalog_only", "domain": "data_boundary", "executable_links": { "backend_registry": false, @@ -2234,6 +2385,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2266,6 +2418,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2298,6 +2451,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2330,6 +2484,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2362,6 +2517,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2394,6 +2550,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2426,6 +2583,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2458,6 +2616,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2493,6 +2652,7 @@ "trace" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2528,6 +2688,7 @@ "trace" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2563,6 +2724,7 @@ "trace" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2597,6 +2759,7 @@ "claimed_backends": [ "deployment-deterministic" ], + "conformance_status_v2": "executable_advisory", "domain": "deployment", "executable_links": { "backend_registry": true, @@ -2611,12 +2774,24 @@ "lane": "deployment", "missing_executable_links": [], "notes": [ - "all required executable links present" + "all required executable links present", + "source_profile_evidence:incomplete" ], "path": "templates/deployment/no_skipped_approval_state.intent.json", "production_status": "strict_eligible", "property_kind": "invariant", "risk_severity": "high", + "source_profile_evidence": { + "coverage_complete": false, + "enforcement_test_present": true, + "materials_trusted": false, + "notes": [ + "requires explicit trusted_profile material for strictness" + ], + "profile_id": "deployment.trusted_profile_v1", + "strict_eligible": false + }, + "source_profile_id": "deployment.trusted_profile_v1", "version": "0.1.0" }, { @@ -2626,6 +2801,7 @@ "claimed_backends": [ "tla" ], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2664,6 +2840,7 @@ "claimed_backends": [ "tla" ], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2702,6 +2879,7 @@ "claimed_backends": [ "tla" ], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2739,6 +2917,7 @@ "claimed_backends": [ "tla" ], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2777,6 +2956,7 @@ "claimed_backends": [ "tla" ], + "conformance_status_v2": "catalog_only", "domain": "deployment", "executable_links": { "backend_registry": false, @@ -2815,6 +2995,7 @@ "claimed_backends": [ "alloy" ], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -2853,6 +3034,7 @@ "claimed_backends": [ "alloy" ], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -2891,6 +3073,7 @@ "claimed_backends": [ "alloy" ], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -2928,6 +3111,7 @@ "claimed_backends": [ "alloy" ], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -2961,6 +3145,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -2993,6 +3178,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3025,6 +3211,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3057,6 +3244,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3089,6 +3277,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3121,6 +3310,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3153,6 +3343,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3185,6 +3376,7 @@ { "acceptable_evidence_kinds": [], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3220,6 +3412,7 @@ "topology_model" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3255,6 +3448,7 @@ "topology_model" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3290,6 +3484,7 @@ "topology_model" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3325,6 +3520,7 @@ "topology_model" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3359,6 +3555,7 @@ "policy_check" ], "claimed_backends": [], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3395,6 +3592,7 @@ "claimed_backends": [ "alloy" ], + "conformance_status_v2": "catalog_only", "domain": "infrastructure", "executable_links": { "backend_registry": false, @@ -3433,6 +3631,7 @@ "claimed_backends": [ "infrastructure-deterministic" ], + "conformance_status_v2": "source_profile_strict_eligible", "domain": "infrastructure", "executable_links": { "backend_registry": true, @@ -3447,12 +3646,25 @@ "lane": "infrastructure", "missing_executable_links": [], "notes": [ - "all required executable links present" + "all required executable links present", + "source_profile_evidence:strict_ok" ], "path": "templates/infrastructure/no_public_sensitive_resource.intent.json", "production_status": "strict_eligible", "property_kind": "data_boundary", "risk_severity": "high", + "source_profile_evidence": { + "coverage_complete": true, + "enforcement_test_present": true, + "materials_trusted": true, + "notes": [ + "resources=1", + "eligibility=strict" + ], + "profile_id": "infrastructure.terraform.plan_recursive_v1", + "strict_eligible": true + }, + "source_profile_id": "infrastructure.terraform.plan_recursive_v1", "version": "0.1.0" } ] diff --git a/ovk/core/template_conformance.py b/ovk/core/template_conformance.py index a9c7bfe..bd3157e 100644 --- a/ovk/core/template_conformance.py +++ b/ovk/core/template_conformance.py @@ -3,6 +3,9 @@ A template is ``catalog_only`` unless every required executable link exists. Public executable claims that lack a compiler/adapter/example/test path are downgraded honestly rather than advertised as strict-eligible. + +PR10 / Sprint 7 semantic statuses (``conformance_status_v2``) derive from +executed source-profile evidence, not mere file presence. """ from __future__ import annotations @@ -14,6 +17,11 @@ from typing import Any, Literal from ovk.core.json_io import read_json_file +from ovk.core.source_profile_evidence import ( + ProfileSemanticEvidence, + collect_source_profile_evidence, +) +from ovk.core.source_profiles import source_profile_strict_eligible ProductionStatus = Literal[ "catalog_only", @@ -23,6 +31,14 @@ "deprecated", ] +ProductionStatusV2 = Literal[ + "catalog_only", + "executable_advisory", + "source_profile_strict_eligible", + "externally_calibrated_strict", + "deprecated", +] + REQUIRED_ROW_FIELDS = ( "intent_id", "path", @@ -54,6 +70,7 @@ EXECUTABLE_CATALOG: dict[str, dict[str, Any]] = { "no-admin-route-bypass": { "lane": "authorization", + "source_profile_id": "authorization.fastapi.ast_v1", "claimed_backends": ["z3-native", "authorization-deterministic"], "links": { "lane_evaluator": "ovk/adapters/authorization/deterministic_adapter.py", @@ -67,6 +84,8 @@ }, "agent-cannot-disable-own-ci-gate": { "lane": "self_protection", + # No source-profile prover yet; stays executable_advisory under v2. + "source_profile_id": None, "claimed_backends": ["opa-native", "self-protection-deterministic"], "links": { "lane_evaluator": "ovk/adapters/self_protection/deterministic_adapter.py", @@ -80,6 +99,7 @@ }, "no-public-sensitive-resource": { "lane": "infrastructure", + "source_profile_id": "infrastructure.terraform.plan_recursive_v1", "claimed_backends": ["infrastructure-deterministic"], "links": { "lane_evaluator": "ovk/adapters/infrastructure/deterministic_adapter.py", @@ -93,6 +113,7 @@ }, "no-secrets-in-untrusted-context": { "lane": "ci_secrets", + "source_profile_id": "ci_secrets.actions.permissions_flow_v1", "claimed_backends": ["ci-secrets-deterministic"], "links": { "lane_evaluator": "ovk/adapters/ci_secrets/deterministic_adapter.py", @@ -106,6 +127,7 @@ }, "no-skipped-approval-state": { "lane": "deployment", + "source_profile_id": "deployment.trusted_profile_v1", "claimed_backends": ["deployment-deterministic"], "links": { "lane_evaluator": "ovk/adapters/deployment/deterministic_adapter.py", @@ -143,6 +165,14 @@ "strict_eligible": 4, } +STATUS_RANK_V2 = { + "deprecated": 0, + "catalog_only": 1, + "executable_advisory": 2, + "source_profile_strict_eligible": 3, + "externally_calibrated_strict": 4, +} + @dataclass(frozen=True) class TemplateConformanceRow: @@ -159,14 +189,46 @@ class TemplateConformanceRow: missing_executable_links: list[str] lane: str | None notes: list[str] + source_profile_id: str | None = None + profile_evidence: ProfileSemanticEvidence | None = None + externally_calibrated: bool = False + + def semantic_status_v2(self) -> ProductionStatusV2: + """Derive PR10 semantic status from executed evidence, not file presence alone.""" + if self.production_status == "deprecated": + return "deprecated" + if self.externally_calibrated and self.profile_evidence is not None: + evidence = self.profile_evidence + if source_profile_strict_eligible( + profile_id=evidence.profile_id, + materials_trusted=evidence.materials_trusted, + coverage_complete=evidence.coverage_complete, + enforcement_test_present=evidence.enforcement_test_present, + ): + return "externally_calibrated_strict" + if self.profile_evidence is not None: + evidence = self.profile_evidence + if source_profile_strict_eligible( + profile_id=evidence.profile_id, + materials_trusted=evidence.materials_trusted, + coverage_complete=evidence.coverage_complete, + enforcement_test_present=evidence.enforcement_test_present, + ): + return "source_profile_strict_eligible" + if self.production_status in {"strict_eligible", "advisory", "experimental"} and not self.missing_executable_links: + return "executable_advisory" + if self.production_status in {"advisory", "experimental"}: + return "executable_advisory" + return "catalog_only" def to_dict(self) -> dict[str, Any]: - return { + payload = { "intent_id": self.intent_id, "path": self.path, "domain": self.domain, "version": self.version, "production_status": self.production_status, + "conformance_status_v2": self.semantic_status_v2(), "risk_severity": self.risk_severity, "property_kind": self.property_kind, "acceptable_evidence_kinds": list(self.acceptable_evidence_kinds), @@ -176,6 +238,11 @@ def to_dict(self) -> dict[str, Any]: "lane": self.lane, "notes": list(self.notes), } + if self.source_profile_id: + payload["source_profile_id"] = self.source_profile_id + if self.profile_evidence is not None: + payload["source_profile_evidence"] = self.profile_evidence.as_dict() + return payload def _infer_claimed_backends(intent_id: str, title: str, catalog_entry: dict[str, Any] | None) -> list[str]: @@ -211,6 +278,7 @@ def classify_template( repo_root: Path, intent_path: Path, template: dict[str, Any], + profile_evidence: ProfileSemanticEvidence | None = None, ) -> TemplateConformanceRow: """Classify one template into an honest production status.""" intent_id = str(template.get("intent_id") or intent_path.stem) @@ -224,6 +292,9 @@ def classify_template( ) notes: list[str] = [] lane = str(catalog_entry["lane"]) if catalog_entry else None + source_profile_id = None + if catalog_entry and catalog_entry.get("source_profile_id"): + source_profile_id = str(catalog_entry["source_profile_id"]) if not missing: status: ProductionStatus = str(catalog_entry.get("max_status", "strict_eligible")) # type: ignore[assignment] @@ -249,6 +320,12 @@ def classify_template( status = "deprecated" notes.append("template marked deprecated") + if profile_evidence is not None: + notes.append( + "source_profile_evidence:" + + ("strict_ok" if profile_evidence.as_dict()["strict_eligible"] else "incomplete") + ) + relative = intent_path.relative_to(repo_root).as_posix() if intent_path.is_absolute() else intent_path.as_posix() evidence = template.get("acceptable_evidence") or [] evidence_kinds = sorted( @@ -260,6 +337,7 @@ def classify_template( ) risk = template.get("risk") if isinstance(template.get("risk"), dict) else {} prop = template.get("property") if isinstance(template.get("property"), dict) else {} + externally_calibrated = bool(template.get("externally_calibrated") is True) return TemplateConformanceRow( intent_id=intent_id, path=relative, @@ -274,18 +352,34 @@ def classify_template( missing_executable_links=missing, lane=lane, notes=notes, + source_profile_id=source_profile_id, + profile_evidence=profile_evidence, + externally_calibrated=externally_calibrated, ) def build_conformance_matrix(repo_root: Path, templates_dir: Path | None = None) -> dict[str, Any]: """Scan templates and build the conformance matrix document.""" templates_dir = templates_dir or (repo_root / "templates") + profile_evidence = collect_source_profile_evidence( + repo_root, + catalog_by_intent=EXECUTABLE_CATALOG, + ) rows: list[TemplateConformanceRow] = [] for path in sorted(templates_dir.rglob("*.intent.json")): template = read_json_file(path) - rows.append(classify_template(repo_root=repo_root, intent_path=path, template=template)) + intent_id = str(template.get("intent_id") or path.stem) + rows.append( + classify_template( + repo_root=repo_root, + intent_path=path, + template=template, + profile_evidence=profile_evidence.get(intent_id), + ) + ) by_status = Counter(row.production_status for row in rows) + by_status_v2 = Counter(row.semantic_status_v2() for row in rows) by_domain = Counter(row.domain for row in rows) payload = { "schema_version": "ovk.template_conformance.v1", @@ -293,8 +387,13 @@ def build_conformance_matrix(repo_root: Path, templates_dir: Path | None = None) "required_row_fields": list(REQUIRED_ROW_FIELDS), "required_executable_links": list(REQUIRED_EXECUTABLE_LINKS), "production_statuses": list(STATUS_RANK.keys()), + "conformance_statuses_v2": list(STATUS_RANK_V2.keys()), "counts_by_status": dict(sorted(by_status.items())), + "counts_by_status_v2": dict(sorted(by_status_v2.items())), "counts_by_domain": dict(sorted(by_domain.items())), + "source_profile_evidence": { + intent: item.as_dict() for intent, item in sorted(profile_evidence.items()) + }, "templates": [row.to_dict() for row in rows], } return payload @@ -309,6 +408,7 @@ def validate_matrix(matrix: dict[str, Any]) -> list[str]: if not isinstance(templates, list) or not templates: failures.append("templates must be a non-empty list") return failures + allowed_v2 = set(STATUS_RANK_V2) for index, row in enumerate(templates): if not isinstance(row, dict): failures.append(f"templates[{index}] must be an object") @@ -319,6 +419,21 @@ def validate_matrix(matrix: dict[str, Any]) -> list[str]: status = row.get("production_status") if status not in STATUS_RANK: failures.append(f"templates[{index}] invalid production_status {status!r}") + status_v2 = row.get("conformance_status_v2") + if status_v2 is not None and status_v2 not in allowed_v2: + failures.append(f"templates[{index}] invalid conformance_status_v2 {status_v2!r}") + if status_v2 == "externally_calibrated_strict": + evidence = row.get("source_profile_evidence") or {} + if not evidence.get("strict_eligible"): + failures.append( + f"{row.get('intent_id')}: externally_calibrated_strict requires strict profile evidence" + ) + if status_v2 == "source_profile_strict_eligible": + evidence = row.get("source_profile_evidence") or {} + if not evidence.get("strict_eligible"): + failures.append( + f"{row.get('intent_id')}: source_profile_strict_eligible requires executed profile evidence" + ) missing = row.get("missing_executable_links") or [] if status in {"strict_eligible", "advisory"} and missing and status == "strict_eligible": failures.append( diff --git a/scripts/build_template_conformance.py b/scripts/build_template_conformance.py index 5a172e0..b4ba85b 100644 --- a/scripts/build_template_conformance.py +++ b/scripts/build_template_conformance.py @@ -49,7 +49,8 @@ def main() -> int: print( f"template conformance: {matrix['template_count']} templates -> {output}" f" (strict_eligible={matrix['counts_by_status'].get('strict_eligible', 0)}," - f" catalog_only={matrix['counts_by_status'].get('catalog_only', 0)})" + f" catalog_only={matrix['counts_by_status'].get('catalog_only', 0)}," + f" source_profile_strict_eligible={matrix.get('counts_by_status_v2', {}).get('source_profile_strict_eligible', 0)})" ) if failures: for failure in failures: diff --git a/tests/test_template_conformance.py b/tests/test_template_conformance.py index 5ca2039..7ee5d9b 100644 --- a/tests/test_template_conformance.py +++ b/tests/test_template_conformance.py @@ -93,3 +93,21 @@ def test_write_and_check_round_trip(tmp_path: Path) -> None: assert validate_matrix(matrix) == [] loaded = json.loads(output.read_text(encoding="utf-8")) assert loaded["template_count"] == matrix["template_count"] + + +def test_semantic_v2_statuses_from_profile_evidence() -> None: + repo = Path(__file__).resolve().parents[1] + matrix = build_conformance_matrix(repo) + assert set(matrix["conformance_statuses_v2"]) == { + "catalog_only", + "executable_advisory", + "source_profile_strict_eligible", + "externally_calibrated_strict", + "deprecated", + } + assert matrix["counts_by_status_v2"].get("externally_calibrated_strict", 0) == 0 + assert matrix["counts_by_status_v2"].get("source_profile_strict_eligible", 0) >= 1 + assert matrix["counts_by_status_v2"].get("executable_advisory", 0) >= 1 + by_id = {row["intent_id"]: row for row in matrix["templates"]} + assert by_id["no-admin-route-bypass"]["conformance_status_v2"] == "source_profile_strict_eligible" + assert by_id["agent-cannot-disable-own-ci-gate"]["conformance_status_v2"] == "executable_advisory" From 5627a09c392c567bbda9aa40efc809c04f76800f Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:54:20 -0700 Subject: [PATCH 23/26] Document label-separated holdout prediction digests. Keep prediction generation free of protected labels and provide a digest helper so evaluation can stay separated from publication aggregates. --- docs/HOLDOUT_LABEL_SEPARATION.md | 35 +++++++++ scripts/digest_holdout_predictions.py | 105 ++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 docs/HOLDOUT_LABEL_SEPARATION.md create mode 100644 scripts/digest_holdout_predictions.py diff --git a/docs/HOLDOUT_LABEL_SEPARATION.md b/docs/HOLDOUT_LABEL_SEPARATION.md new file mode 100644 index 0000000..e154750 --- /dev/null +++ b/docs/HOLDOUT_LABEL_SEPARATION.md @@ -0,0 +1,35 @@ +# Label-Separated Holdout Evaluation (Sprint 8) + +Checklist for R2 Sprint 8. Builds on Phase A FormalPR-Holdout isolation +(`scripts/run_formalpr_holdout.py`, `.github/workflows/holdout-eval.yml`). + +## Required flow + +1. **Predict** using the exact RC artifact (wheel / Action pin) **without** access to protected labels. +2. **Digest** the predictions file (`scripts/digest_holdout_predictions.py`) — refuses embedded labels / ground-truth fields; emits SHA-256 record. +3. **Evaluate separately** with protected labels (token only on download step; evaluator env token-free). +4. **Publish aggregates only** (`formalpr_holdout.aggregate_metrics.v1`). + +## In-repo artifacts + +| Item | Path / note | +|---|---| +| Runner | `scripts/run_formalpr_holdout.py` (requires `--asset-sha256`; validates predictions are label-free) | +| Predictions digest | `scripts/digest_holdout_predictions.py` | +| Workflow | `.github/workflows/holdout-eval.yml` (download vs eval token split) | +| Predictions placeholder | `.verification/holdout-predictions.json` (never commit labels) | +| Governance | [FORMALPR_HOLDOUT_GOVERNANCE.md](FORMALPR_HOLDOUT_GOVERNANCE.md) | + +## Checklist + +- [ ] RC predictions generated in an environment without `corpus/labels` +- [ ] Predictions digested (`digest_holdout_predictions.py`) and retained with workflow ID +- [ ] `HOLDOUT_ASSET_SHA256` (or workflow input) set for immutable asset verify +- [ ] Eval job runs with tokens unset; aggregates schema-validated +- [ ] Published metrics cite `ovk_commit_sha` / `benchmark_source_sha` and do not embed case ids +- [ ] Do not set `verified_source_sha` on holdout aggregates unless the full required-workflow set was observed + +## Blocked outside this repo + +Protected label store and annotator workflow live in `fraware/FormalPR-Holdout` (private). +This repository cannot complete live holdout scoring without that access. diff --git a/scripts/digest_holdout_predictions.py b/scripts/digest_holdout_predictions.py new file mode 100644 index 0000000..a8b0467 --- /dev/null +++ b/scripts/digest_holdout_predictions.py @@ -0,0 +1,105 @@ +"""Digest and validate label-free holdout predictions (Sprint 8). + +Predictions are produced from the RC artifact without protected labels, then +digested (and optionally signed) before a separate evaluator consumes them. + +Case ids may appear in predictions (needed for scoring). Protected *labels* / +ground-truth fields must not. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any + +_LABEL_FORBIDDEN_SUBSTRINGS = ( + "expected_status", + "expected_merge_recommendation", + "ground_truth_class", + "true_positive_unsafe", + "true_negative_safe", + "corpus/labels", + "labels_dir", + "ground_truth", + "expected_outcome", +) + +_TOP_LEVEL_FORBIDDEN = frozenset( + { + "labels", + "expected_status", + "ground_truth_class", + "corpus_labels", + "expected_merge_recommendation", + } +) + +_CASE_FORBIDDEN = frozenset( + { + "expected_status", + "ground_truth_class", + "label", + "labels", + "expected_merge_recommendation", + } +) + + +def _fail(msg: str) -> None: + raise SystemExit(f"fail-closed: {msg}") + + +def assert_predictions_label_free(payload: Any) -> None: + """Refuse predictions that embed protected labels or case-ground-truth fields.""" + text = json.dumps(payload, sort_keys=True) + for token in _LABEL_FORBIDDEN_SUBSTRINGS: + if token in text: + _fail(f"predictions contain protected token {token!r}") + if isinstance(payload, dict): + for key in _TOP_LEVEL_FORBIDDEN: + if key in payload: + _fail(f"predictions must not include top-level key {key!r}") + cases = payload.get("cases") or payload.get("predictions") + if isinstance(cases, list): + for index, item in enumerate(cases): + if not isinstance(item, dict): + continue + for key in _CASE_FORBIDDEN: + if key in item: + _fail(f"predictions[{index}] must not include {key!r}") + + +def digest_predictions_file(path: Path) -> dict[str, Any]: + raw = path.read_bytes() + payload = json.loads(raw.decode("utf-8")) + assert_predictions_label_free(payload) + digest = hashlib.sha256(raw).hexdigest() + return { + "schema_version": "ovk.holdout.predictions_digest.v1", + "predictions_path": path.as_posix(), + "sha256": digest, + "byte_length": len(raw), + "label_free": True, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Digest label-free holdout predictions") + parser.add_argument("--predictions", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + if not args.predictions.is_file(): + _fail(f"predictions file not found: {args.predictions}") + record = digest_predictions_file(args.predictions) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + print(f"predictions digest ok: sha256={record['sha256']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From af275648aa16a3c206a0ce82d27265198afa4ad3 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:54:20 -0700 Subject: [PATCH 24/26] Refresh consumer validation checklist for rc.1 pins. Update immutable-pin scaffolding and workflow evidence collection so external consumer gates target current typed-control-plane candidates instead of v1.2.1 alone. --- docs/CONSUMER_VALIDATION_CHECKLIST.md | 144 ++++++++++++------ .../consumer_validation.workflow.yml | 18 ++- scripts/collect_workflow_evidence.py | 109 +++++++++++++ 3 files changed, 223 insertions(+), 48 deletions(-) create mode 100644 scripts/collect_workflow_evidence.py diff --git a/docs/CONSUMER_VALIDATION_CHECKLIST.md b/docs/CONSUMER_VALIDATION_CHECKLIST.md index a046634..da4b213 100644 --- a/docs/CONSUMER_VALIDATION_CHECKLIST.md +++ b/docs/CONSUMER_VALIDATION_CHECKLIST.md @@ -1,46 +1,98 @@ -# Consumer Validation Checklist - -Scaffolding and live pointers for independent consumer repositories validating OVK. -Completing this checklist for one repo does **not** satisfy the multi-repo production -exit criterion (30 human-adjudicated PRs per independent consumer). - -## Live independent consumers (v1.2.1) - -| Repository | Stack | Ledger | -|---|---|---| -| [fraware/ovk-consumer-fastapi-terraform](https://github.com/fraware/ovk-consumer-fastapi-terraform) | FastAPI + Terraform | [`pilot/ledger.json`](https://github.com/fraware/ovk-consumer-fastapi-terraform/blob/main/pilot/ledger.json) | -| [fraware/ovk-consumer-express-actions](https://github.com/fraware/ovk-consumer-express-actions) | Express + GitHub Actions | [`pilot/ledger.json`](https://github.com/fraware/ovk-consumer-express-actions/blob/main/pilot/ledger.json) | - -Both pin `fraware/open-verification-kernel@v1.2.1` (never `uses: ./`). CI fails on pin drift via `scripts/assert_ovk_pin.py`. - -## Immutable pin requirements - -Consumers must pin an **immutable** OVK commit SHA or release tag. - -```yaml -env: - OVK_PACKAGE_VERSION: "1.2.1" -steps: - - uses: fraware/open-verification-kernel@v1.2.1 -``` - -Forbidden: - -- `uses: ./` -- `uses: fraware/open-verification-kernel@main` -- floating refs without a tag or full commit SHA - -## Checklist (per consumer) - -- [x] Workflow copies from `docs/templates/consumer_validation.workflow.yml` (or equivalent) with an immutable pin. -- [x] Automated scenario matrix covers program section 23.1 intents (see consumer README). -- [x] Adjudication rows recorded in a pilot ledger conforming to `schemas/pilot.ledger.schema.json`. -- [ ] Human adjudications reach 30 PRs (entries must not remain `automated_scenario` / `pending` only). -- [ ] True cross-fork PR exercised and ledger-adjudicated (see consumer `docs/FORK_PR.md`). -- [ ] Prefer PyPI `open-verification-kernel==1.2.1` once published; until then Release wheel + cosign verify-blob. - -## What this does not claim - -- Declaring two independent consumer repos with 30 adjudicated PRs complete -- Vision completion or Production-stable package status -- That FormalPR-Holdout results generalize to these consumers (holdout is a separate program) +# Consumer Validation Checklist + +Scaffolding and live pointers for independent consumer repositories validating OVK. +Completing this checklist for one repo does **not** satisfy the multi-repo production +exit criterion (30 human-adjudicated PRs per independent consumer). + +## Live independent consumers + +| Repository | Stack | Current pin | Target pin (Sprint 9) | +|---|---|---|---| +| [fraware/ovk-consumer-fastapi-terraform](https://github.com/fraware/ovk-consumer-fastapi-terraform) | FastAPI + Terraform | `v1.2.1` | immutable `v1.3.0-rc.1` (or audited commit) | +| [fraware/ovk-consumer-express-actions](https://github.com/fraware/ovk-consumer-express-actions) | Express + GitHub Actions | `v1.2.1` | immutable `v1.3.0-rc.1` (or audited commit) | + +`v1.2.1` validates the **pre-control-plane** signed release only. Typed control-plane +commits must not inherit that consumer evidence. Both consumers use +`scripts/assert_ovk_pin.py` to fail on pin drift. + +## Immutable pin requirements + +Consumers must pin an **immutable** OVK commit SHA or release tag. + +```yaml +env: + OVK_PACKAGE_VERSION: "1.3.0rc1" # after rc.1 cut; until then keep 1.2.1 + OVK_ACTION_REF: "v1.3.0-rc.1" +steps: + - uses: fraware/open-verification-kernel@v1.3.0-rc.1 +``` + +In-repo template: [templates/consumer_validation.workflow.yml](templates/consumer_validation.workflow.yml). + +Forbidden: + +- `uses: ./` +- `uses: fraware/open-verification-kernel@main` +- floating refs without a tag or full commit SHA + +## Maintainer steps after `v1.3.0-rc.1` exists (do not push from this workspace alone) + +For each consumer repository: + +1. Open a pin PR that bumps Action `uses:` and `OVK_PACKAGE_VERSION` to the immutable rc.1 tag (or full SHA). +2. Merge the pin PR (or push to a validation branch) so workflows can see the new pin. +3. Dispatch validation: + ```bash + gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-fastapi-terraform + gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-express-actions + ``` + (Use the exact workflow name as defined in each consumer.) +4. Await conclusions; download evidence artifacts: + ```bash + gh run download --repo -n -D ./consumer-evidence// + ``` +5. Verify bundles with the OVK release verifier + cosign as applicable for the pin. +6. Exercise a true cross-fork PR path (`docs/FORK_PR.md` in each consumer). +7. Update the pilot ledger: keep `automated_scenario` rows distinct from human adjudications. + +Local clone prep (optional, no push): + +```bash +git clone https://github.com/fraware/ovk-consumer-fastapi-terraform.git +git clone https://github.com/fraware/ovk-consumer-express-actions.git +# Edit workflow pins locally; do not git push until maintainers cut rc.1. +``` + +## Sprint 9 checklist (per consumer) — prepare in this repo; land in consumer repos + +In-repo preparation (this repository): + +- [x] Document rc.1 target pins and provenance correction (this checklist + R2 status) +- [x] Keep human pilot ledgers separate from automated fixtures (see consumer `pilot/ledger.json` policy) +- [x] Template workflow targets `v1.3.0-rc.1` (copy only after tag exists) +- [ ] Cut attributable `v1.3.0-rc.1` tag on verified source (Sprint 10) + +In consumer repositories (requires write access — **blocked from this workspace alone**): + +- [ ] Bump Action pin from `v1.2.1` → `v1.3.0-rc.1` (or audited full SHA) +- [ ] Bump `OVK_PACKAGE_VERSION` / wheel install scripts to match +- [ ] Dispatch validation workflows; await conclusions +- [ ] Download evidence bundles; verify with release verifier + cosign as applicable +- [ ] Exercise true cross-fork PR path (`docs/FORK_PR.md`) +- [ ] Update ledger: automated scenarios remain distinct from human adjudications + +## Checklist (per consumer) — ongoing + +- [x] Workflow copies from `docs/templates/consumer_validation.workflow.yml` (or equivalent) with an immutable pin. +- [x] Automated scenario matrix covers program section 23.1 intents (see consumer README). +- [x] Adjudication rows recorded in a pilot ledger conforming to `schemas/pilot.ledger.schema.json`. +- [ ] Human adjudications reach 30 PRs (entries must not remain `automated_scenario` / `pending` only). +- [ ] True cross-fork PR exercised and ledger-adjudicated (see consumer `docs/FORK_PR.md`). +- [ ] Prefer PyPI once published; until then Release wheel + cosign verify-blob at the **current** pin. + +## What this does not claim + +- Declaring two independent consumer repos with 30 adjudicated PRs complete +- Vision completion or Production-stable package status +- That FormalPR-Holdout results generalize to these consumers (holdout is a separate program) +- That `v1.2.1` consumer green runs validate typed-control-plane `main` diff --git a/docs/templates/consumer_validation.workflow.yml b/docs/templates/consumer_validation.workflow.yml index a2ac0fc..3bdb389 100644 --- a/docs/templates/consumer_validation.workflow.yml +++ b/docs/templates/consumer_validation.workflow.yml @@ -4,10 +4,15 @@ name: OVK Consumer Validation # Live examples: # https://github.com/fraware/ovk-consumer-fastapi-terraform # https://github.com/fraware/ovk-consumer-express-actions +# +# Sprint 9 target pin: immutable v1.3.0-rc.1 (or audited full SHA) AFTER the +# attributable rc.1 tag exists. Until then, keep the signed v1.2.1 pin. +# Do not push consumer changes from this template alone. on: pull_request: branches: [main] + workflow_dispatch: permissions: contents: read @@ -15,7 +20,9 @@ permissions: checks: write env: - OVK_PACKAGE_VERSION: "1.2.1" + # After rc.1 cut: "1.3.0rc1". Until then keep "1.2.1". + OVK_PACKAGE_VERSION: "1.3.0rc1" + OVK_ACTION_REF: "v1.3.0-rc.1" jobs: ovk-consumer-validation: @@ -24,8 +31,14 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 2 + - name: Guard pin target + run: | + echo "Expected Action ref: ${OVK_ACTION_REF}" + echo "Expected package: ${OVK_PACKAGE_VERSION}" + echo "Refuse floating @main pins. Prefer tag or full commit SHA." - name: Run OVK advisory consumer path - uses: fraware/open-verification-kernel@v1.2.1 + # Replace with the immutable tag/SHA once v1.3.0-rc.1 exists. + uses: fraware/open-verification-kernel@v1.3.0-rc.1 with: mode: advisory use-check: "true" @@ -35,3 +48,4 @@ jobs: run: | echo "Adjudicate this PR into a pilot ledger conforming to schemas/pilot.ledger.schema.json" echo "Do not set production_gate_met=true until 30 human adjudications exist." + echo "Keep automated_scenario rows distinct from human adjudications." diff --git a/scripts/collect_workflow_evidence.py b/scripts/collect_workflow_evidence.py new file mode 100644 index 0000000..7b00d0d --- /dev/null +++ b/scripts/collect_workflow_evidence.py @@ -0,0 +1,109 @@ +"""Collect GitHub Actions workflow run IDs and digests for attributable publication. + +Requires network + ``gh`` auth. Does not push or mutate remotes. +When Actions are unavailable, exits non-zero with an explicit blocker message. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +REQUIRED_WORKFLOW_NAMES = ( + "CI", + "Native Tier 1", + "Release", + "Bench", +) + + +def _run_gh(args: list[str]) -> tuple[int, str, str]: + try: + completed = subprocess.run( + ["gh", *args], + capture_output=True, + text=True, + check=False, + timeout=60, + ) + except (OSError, subprocess.SubprocessError) as exc: + return 1, "", str(exc) + return completed.returncode, completed.stdout, completed.stderr + + +def collect_for_sha(*, repo: str, sha: str, limit: int = 30) -> dict[str, Any]: + code, stdout, stderr = _run_gh( + [ + "run", + "list", + "--repo", + repo, + "--commit", + sha, + "--limit", + str(limit), + "--json", + "databaseId,displayTitle,workflowName,status,conclusion,url,headSha,createdAt", + ] + ) + if code != 0: + return { + "ok": False, + "blocker": "gh_run_list_failed", + "detail": stderr.strip() or stdout.strip() or "gh run list failed", + "benchmark_source_sha": sha, + "verified_source_sha": None, + "collected_at": datetime.now(timezone.utc).isoformat(), + "runs": [], + } + runs = json.loads(stdout or "[]") + by_workflow: dict[str, list[dict[str, Any]]] = {} + for run in runs: + name = str(run.get("workflowName") or "unknown") + by_workflow.setdefault(name, []).append(run) + observed = sorted(by_workflow) + # verified_source_sha is only set when callers confirm the full required set. + return { + "ok": True, + "blocker": None, + "benchmark_source_sha": sha, + "verified_source_sha": None, + "required_workflow_names": list(REQUIRED_WORKFLOW_NAMES), + "observed_workflow_names": observed, + "collected_at": datetime.now(timezone.utc).isoformat(), + "runs": runs, + "note": ( + "Cite benchmark_source_sha for measurement identity. " + "Set verified_source_sha only after maintainers confirm the complete " + "required-workflow set on this commit." + ), + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Collect workflow evidence for a source SHA") + parser.add_argument("--repo", default="fraware/open-verification-kernel") + parser.add_argument("--sha", required=True, help="Source commit SHA") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + payload = collect_for_sha(repo=args.repo, sha=args.sha) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + if not payload.get("ok"): + print(f"blocked: {payload.get('blocker')}: {payload.get('detail')}", file=sys.stderr) + return 2 + print( + f"collected {len(payload.get('runs') or [])} runs for " + f"benchmark_source_sha={args.sha}; verified_source_sha left unset" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 66455294e58a065de5e9db050b4b249314b2a751 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 23 Jul 2026 23:54:21 -0700 Subject: [PATCH 25/26] Add attributable publication gate for rc.1 and v1.3.0. Require correct benchmark vs verified-source terminology, signed artifacts, and workflow retention before promoting the release candidate. --- docs/ATTRIBUTABLE_PUBLICATION.md | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/ATTRIBUTABLE_PUBLICATION.md diff --git a/docs/ATTRIBUTABLE_PUBLICATION.md b/docs/ATTRIBUTABLE_PUBLICATION.md new file mode 100644 index 0000000..83affe7 --- /dev/null +++ b/docs/ATTRIBUTABLE_PUBLICATION.md @@ -0,0 +1,51 @@ +# Attributable Publication Checklist (Sprint 10) + +Gate for publishing **`v1.3.0-rc.1`** and later promoting to **`v1.3.0`**. +Authority: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) 18-condition gate. + +## Terminology + +| Field | Use | +|---|---| +| `benchmark_source_sha` | FormalPR-Bench / badge measurement identity | +| `verified_source_sha` | Complete observed required-workflow set only | + +Never label a `[skip ci]` badge commit as verified. Never re-attribute `v1.2.1` +Sigstore / consumer evidence to typed-control-plane commits. + +## Collect workflow evidence (when Actions are available) + +```bash +python scripts/collect_workflow_evidence.py \ + --sha \ + --output .verification/workflow-evidence-.json +``` + +The collector records run IDs/URLs under `benchmark_source_sha` and leaves +`verified_source_sha` unset until maintainers confirm the full required set. + +## Pre-tag checklist (`v1.3.0-rc.1`) + +- [ ] P0 trust PRs 1–9 landed on the tag source +- [ ] Non-`[skip ci]` CI, native Tier 1, wheel smoke, Action dogfood, release preflight green +- [ ] Expanded FormalPR-Bench recorded with `benchmark_source_sha` +- [ ] Template conformance v2 matrix regenerated from semantic statuses +- [ ] Both consumers dispatched on immutable rc.1 pin (or audited commit); evidence downloaded and verified +- [ ] Label-separated holdout aggregates retained (predictions digest + eval workflow IDs) +- [ ] Release artifacts signed; workflow IDs and digests recorded in [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) + +## Promote to `v1.3.0` + +Only after: + +- [ ] All 18 completion-gate conditions hold +- [ ] P0 closure (PRs 1–9) on the exact tag source +- [ ] Consumer validation on the exact pin +- [ ] Attributable holdout aggregates (predictions digest + eval) +- [ ] Human pilot ledgers remain separate from automated fixtures +- [ ] No re-attribution of `v1.2.1` Sigstore evidence to typed-control-plane commits + +## Blocked without external access + +Live GitHub Actions run URLs, consumer repo pin PRs, and private holdout evaluation require +maintainer credentials outside this working tree. From 90fce18f8e61139e56d40d0e1c3382f72fd57dc5 Mon Sep 17 00:00:00 2001 From: fraware Date: Fri, 24 Jul 2026 01:18:00 -0700 Subject: [PATCH 26/26] Fix ruff formatting after merging main into the R2 branch. Normalize import blocks and blank lines so CI lint passes on the merged tree. --- benchmarks/formal_pr_bench/score_all_lanes.py | 9 +- .../score_authorization_obligation.py | 3 +- benchmarks/formal_pr_bench/scoring.py | 12 +- .../authorization/deterministic_adapter.py | 5 +- ovk/adapters/authorization/z3_adapter.py | 10 +- ovk/adapters/cbmc/diff_extract.py | 4 +- ovk/adapters/cbmc/harness_compiler.py | 5 +- ovk/adapters/cedar/adapter.py | 4 +- .../ci_secrets/deterministic_adapter.py | 7 +- ovk/adapters/ci_secrets/exposure.py | 3 +- ovk/adapters/ci_secrets/regression.py | 11 +- .../deployment/deterministic_adapter.py | 7 +- ovk/adapters/deployment/diff_extract.py | 3 +- ovk/adapters/deployment/regression.py | 11 +- ovk/adapters/deployment/state_machine.py | 3 +- ovk/adapters/infra/graph.py | 4 +- .../infrastructure/deterministic_adapter.py | 8 +- ovk/adapters/lane/_base.py | 4 +- ovk/adapters/opa/infra_exposure.py | 11 +- ovk/adapters/opa/policy_assets.py | 7 +- ovk/adapters/opa/self_protection.py | 4 +- .../self_protection/deterministic_adapter.py | 16 +- ovk/adapters/self_protection/opa_adapter.py | 8 +- ovk/adapters/z3/obligation.py | 5 +- ovk/adapters/z3/regression.py | 6 +- ovk/adapters/z3/smt_plan.py | 5 +- ovk/cli.py | 54 ++- ovk/compilers/authorization/corpus.py | 6 +- ovk/compilers/authorization/express.py | 4 +- ovk/compilers/authorization/fastapi_ast.py | 4 +- ovk/compilers/cbmc/traceability.py | 4 +- ovk/compilers/deployment/ir.py | 5 +- .../github_actions/composite_actions.py | 4 +- ovk/compilers/github_actions/permissions.py | 3 +- .../github_actions/reusable_workflows.py | 4 +- ovk/compilers/github_actions/secrets.py | 5 +- ovk/compilers/github_actions/trust_flow.py | 14 +- .../infrastructure/exposure_graph.py | 4 +- ovk/compilers/infrastructure/kubernetes.py | 9 +- ovk/compilers/infrastructure/reachability.py | 10 +- .../infrastructure/terraform_plan.py | 4 +- ovk/core/adapter_runtime.py | 331 +----------------- ovk/core/backend_aggregation.py | 17 +- ovk/core/backend_control_plane.py | 16 +- ovk/core/backend_ids.py | 12 +- ovk/core/backend_registry.py | 22 +- ovk/core/bundle.py | 4 +- ovk/core/compiler_bridge.py | 3 +- ovk/core/context.py | 9 +- ovk/core/counterexample_translator.py | 11 +- ovk/core/decision.py | 27 +- ovk/core/diff_iac.py | 8 +- ovk/core/doctor.py | 4 +- ovk/core/evidence_from_execution.py | 163 +-------- ovk/core/evidence_invariants.py | 6 +- ovk/core/infrastructure_compiler.py | 3 +- ovk/core/lane_compiler.py | 16 +- ovk/core/multi_lane.py | 14 +- ovk/core/native_backend_probe.py | 4 +- ovk/core/obligation_compiler.py | 11 +- ovk/core/output_validation.py | 8 +- ovk/core/policy_config.py | 4 +- ovk/core/provenance.py | 8 +- ovk/core/release_preflight_report.py | 16 +- ovk/core/render.py | 10 +- ovk/core/result_cache.py | 1 - ovk/core/router.py | 15 +- ovk/core/routing_pipeline.py | 5 +- ovk/core/schema_validation.py | 3 +- ovk/core/sigstore_signing.py | 4 +- ovk/core/source_profiles.py | 7 +- ovk/core/template_conformance.py | 25 +- ovk/mcp_server.py | 6 +- scripts/expand_ovk_library.py | 85 ++++- scripts/render_bench_badge.py | 4 +- scripts/render_pilot_metrics.py | 4 +- scripts/run_formalpr_holdout.py | 4 +- scripts/sync_real_diff_cases.py | 1 + scripts/write_artifact_manifest.py | 7 +- tests/test_adapter_isolation_r2_pr8.py | 14 +- tests/test_adversarial_control_plane.py | 16 +- tests/test_auth_obligation_cli.py | 1 - tests/test_authorization_compilers.py | 5 +- tests/test_authorization_enforcement.py | 8 +- tests/test_authorization_input_schema.py | 8 +- tests/test_backends.py | 8 +- tests/test_cache_worker_control_plane.py | 8 +- tests/test_change_detection.py | 12 +- tests/test_compiler_runtime_wiring.py | 4 +- tests/test_decision.py | 5 +- tests/test_deterministic_worker_r2_pr7.py | 9 +- tests/test_emit_github_check.py | 46 +-- tests/test_infra_graph.py | 4 +- tests/test_ingest_external_pilot_metrics.py | 10 +- tests/test_material_reference_sizes.py | 2 +- tests/test_native_backends.py | 5 +- tests/test_opa_infra_domain.py | 8 +- tests/test_phase3_trust.py | 4 +- tests/test_phase7_v1.py | 3 +- tests/test_policy_loading_fail_closed.py | 4 +- tests/test_render_bundle.py | 8 +- tests/test_render_pilot_metrics.py | 22 +- tests/test_repair_loop.py | 4 +- tests/test_repo_memory_trust.py | 4 +- tests/test_result_cache_semantics.py | 18 +- tests/test_routing_pipeline_r2_pr6.py | 7 +- tests/test_runtime_cache_regimes.py | 4 +- tests/test_scripts_sprint2.py | 10 +- tests/test_self_protection_enforcement.py | 19 +- tests/test_self_protection_sprint1.py | 4 +- tests/test_shadow_control_plane.py | 4 +- tests/test_sigstore_release.py | 14 +- tests/test_source_profile_hardening.py | 31 +- tests/test_typed_router.py | 7 +- tests/test_verification_cache.py | 4 +- tests/test_z3_obligations.py | 8 +- tests/test_z3_validation.py | 5 +- 117 files changed, 516 insertions(+), 1048 deletions(-) diff --git a/benchmarks/formal_pr_bench/score_all_lanes.py b/benchmarks/formal_pr_bench/score_all_lanes.py index 279df5f..bddc8b8 100644 --- a/benchmarks/formal_pr_bench/score_all_lanes.py +++ b/benchmarks/formal_pr_bench/score_all_lanes.py @@ -43,7 +43,9 @@ def evaluate_lane_case(case: dict) -> tuple[str, str, str | None]: if intent == "agent-cannot-disable-own-ci-gate": from ovk.adapters.opa import evaluate_self_protection - evidence = evaluate_self_protection(json.loads(Path(fixture).read_text(encoding="utf-8")), repo="bench/repo", head_sha="seed") + evidence = evaluate_self_protection( + json.loads(Path(fixture).read_text(encoding="utf-8")), repo="bench/repo", head_sha="seed" + ) payload = evidence.model_dump(mode="json") elif intent in INTENT_TO_LANE: payload = _evaluate_lane_fixture(INTENT_TO_LANE[intent], fixture) @@ -109,10 +111,7 @@ def run_benchmark( cases, case_set = load_cases(expanded=expanded, include_extended=include_extended) capabilities = CapabilityRegistry.from_directory(ROOT / "adapters").all() lane_evaluator: Callable[[dict[str, Any]], tuple[str, str, str | None]] = evaluate_lane_case - scores = [ - score_case(case, capabilities=capabilities, lane_evaluator=lane_evaluator) - for case in cases - ] + scores = [score_case(case, capabilities=capabilities, lane_evaluator=lane_evaluator) for case in cases] leaderboard = build_leaderboard( scores, benchmark_name="FormalPR-Bench", diff --git a/benchmarks/formal_pr_bench/score_authorization_obligation.py b/benchmarks/formal_pr_bench/score_authorization_obligation.py index 74f7a04..2480c7b 100644 --- a/benchmarks/formal_pr_bench/score_authorization_obligation.py +++ b/benchmarks/formal_pr_bench/score_authorization_obligation.py @@ -32,8 +32,7 @@ def score_malformed_fixture(path: Path) -> bool: status_ok = evidence.backend_claims[0].status.value == "unknown" decision_ok = evidence.decision["merge_recommendation"] == "require_human_review" diagnostic_ok = any( - item.get("failure_mode") == "authorization_abstraction_invalid" - for item in evidence.counterexamples + item.get("failure_mode") == "authorization_abstraction_invalid" for item in evidence.counterexamples ) return status_ok and decision_ok and diagnostic_ok diff --git a/benchmarks/formal_pr_bench/scoring.py b/benchmarks/formal_pr_bench/scoring.py index 52d6439..5317dff 100644 --- a/benchmarks/formal_pr_bench/scoring.py +++ b/benchmarks/formal_pr_bench/scoring.py @@ -113,9 +113,7 @@ def score_routing_case(case: dict[str, Any], *, elapsed_ms: float) -> DimensionS def score_quality_case(case: dict[str, Any], *, elapsed_ms: float) -> DimensionScore: """Score evidence honesty via the quality gate on a bundle fixture.""" - bundle = EvidenceBundle.model_validate( - json.loads((ROOT / case["input_fixture"]).read_text(encoding="utf-8")) - ) + bundle = EvidenceBundle.model_validate(json.loads((ROOT / case["input_fixture"]).read_text(encoding="utf-8"))) report = build_evidence_quality_report(bundle) expected_pass = bool(case.get("expected_quality_passed", False)) honest = report.passed == expected_pass @@ -143,9 +141,7 @@ def score_repair_loop_case(case: dict[str, Any], *, elapsed_ms: float) -> Dimens recommendation = str(result.bundle.decision.get("merge_recommendation", "unknown")) merge_ok = recommendation == case["expected_merge_recommendation"] counterexamples = [ - counterexample - for evidence in result.bundle.evidence - for counterexample in evidence.counterexamples + counterexample for evidence in result.bundle.evidence for counterexample in evidence.counterexamples ] hints = [repair_hint_for_counterexample(item) for item in counterexamples] expected_fix = case.get("expected_fix_class") @@ -343,9 +339,7 @@ def _rate(values: list[bool | None]) -> float | None: "evidence_honesty": _rate([score.evidence_honest for score in scores]), "intent_recall": _rate([score.status_correct for score in scores if score.category == "intent_recall"]), "real_diff_recall": _rate([score.status_correct for score in scores if score.category == "real_diff"]), - "real_diff_intent_recall": _rate( - [score.status_correct for score in scores if score.category == "real_diff"] - ), + "real_diff_intent_recall": _rate([score.status_correct for score in scores if score.category == "real_diff"]), "by_category": { category: { "cases_total": len(items), diff --git a/ovk/adapters/authorization/deterministic_adapter.py b/ovk/adapters/authorization/deterministic_adapter.py index f2f95b1..ef20049 100644 --- a/ovk/adapters/authorization/deterministic_adapter.py +++ b/ovk/adapters/authorization/deterministic_adapter.py @@ -2,7 +2,6 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any @@ -140,9 +139,7 @@ def compile( environment_requirements={"native": False}, expected_guarantee="deterministic_witness", ) - return provisional.model_copy( - update={"backend_obligation_id": compute_backend_obligation_id(provisional)} - ) + return provisional.model_copy(update={"backend_obligation_id": compute_backend_obligation_id(provisional)}) def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironmentFingerprint: return BackendEnvironmentFingerprint( diff --git a/ovk/adapters/authorization/z3_adapter.py b/ovk/adapters/authorization/z3_adapter.py index ef6d219..c26a81e 100644 --- a/ovk/adapters/authorization/z3_adapter.py +++ b/ovk/adapters/authorization/z3_adapter.py @@ -2,14 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.z3.executor import run_authorization_obligation_with_z3 -from ovk.adapters.z3.obligation import build_authorization_obligation -from ovk.adapters.z3.result import normalize_z3_authorization_result -from ovk.adapters.z3.validation import validate_authorization_input from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -27,7 +22,6 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus @@ -162,9 +156,7 @@ def compile( environment_requirements={"native": True, "binary": "z3-solver"}, expected_guarantee="smt_refutation_search", ) - return provisional.model_copy( - update={"backend_obligation_id": compute_backend_obligation_id(provisional)} - ) + return provisional.model_copy(update={"backend_obligation_id": compute_backend_obligation_id(provisional)}) def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironmentFingerprint: native = z3_available() diff --git a/ovk/adapters/cbmc/diff_extract.py b/ovk/adapters/cbmc/diff_extract.py index 5df2b7c..770e01c 100644 --- a/ovk/adapters/cbmc/diff_extract.py +++ b/ovk/adapters/cbmc/diff_extract.py @@ -23,9 +23,7 @@ def _findings_from_c(content: str, *, intent_id: str) -> list[dict[str, str]]: findings: list[dict[str, str]] = [] if intent_id == "cbmc-no-integer-overflow-quota" and re.search(r"\+\s*=|\+\+|quota|limit", content): findings.append({"kind": "integer_overflow", "summary": "quota or counter arithmetic changed in diff"}) - if intent_id == "cbmc-no-unchecked-buffer-copy" and re.search( - r"\b(memcpy|strcpy|strncpy|memmove)\s*\(", content - ): + if intent_id == "cbmc-no-unchecked-buffer-copy" and re.search(r"\b(memcpy|strcpy|strncpy|memmove)\s*\(", content): findings.append({"kind": "unchecked_copy", "summary": "memory copy operation introduced or modified in diff"}) if intent_id == "cbmc-no-use-after-free-auth-cache" and re.search(r"\bfree\s*\(", content): findings.append({"kind": "use_after_free", "summary": "free() call introduced or modified in diff"}) diff --git a/ovk/adapters/cbmc/harness_compiler.py b/ovk/adapters/cbmc/harness_compiler.py index 2e1a751..71ffbd6 100644 --- a/ovk/adapters/cbmc/harness_compiler.py +++ b/ovk/adapters/cbmc/harness_compiler.py @@ -74,10 +74,7 @@ def _generated_integer_overflow_harness(data: dict[str, Any]) -> str: expect_violation = bool(data.get("expect_violation", data.get("failed_assertions"))) guard = "" if not expect_violation: - guard = ( - f" __CPROVER_assume(used <= {quota_limit});\n" - f" __CPROVER_assume(delta <= {quota_limit} - used);\n" - ) + guard = f" __CPROVER_assume(used <= {quota_limit});\n __CPROVER_assume(delta <= {quota_limit} - used);\n" return f"""#include #include diff --git a/ovk/adapters/cedar/adapter.py b/ovk/adapters/cedar/adapter.py index 968386e..fc530b6 100644 --- a/ovk/adapters/cedar/adapter.py +++ b/ovk/adapters/cedar/adapter.py @@ -58,7 +58,9 @@ def evaluate_evidence( "The decision is produced by the deterministic Cedar-shaped input oracle.", ] if binary_present: - assumptions.append("The Cedar CLI version probe passed; no policy evaluation was executed by the native tool.") + assumptions.append( + "The Cedar CLI version probe passed; no policy evaluation was executed by the native tool." + ) else: assumptions.append("The Cedar CLI was unavailable; native policy evaluation was not attempted.") diff --git a/ovk/adapters/ci_secrets/deterministic_adapter.py b/ovk/adapters/ci_secrets/deterministic_adapter.py index 11de591..b1e3659 100644 --- a/ovk/adapters/ci_secrets/deterministic_adapter.py +++ b/ovk/adapters/ci_secrets/deterministic_adapter.py @@ -2,11 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.ci_secrets.exposure import find_ci_secrets_counterexamples from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -24,7 +22,6 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus @@ -193,9 +190,7 @@ def normalize( assumptions=["Deterministic CI secrets boundary evaluator."], limits=["Does not expand remote reusable workflows."], counterexamples=list(raw.raw_result.get("counterexamples") or []), - generated_artifacts=[ - {"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False} - ], + generated_artifacts=[{"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False}], ) def explain(self, result: NormalizedBackendResult) -> HumanExplanation: diff --git a/ovk/adapters/ci_secrets/exposure.py b/ovk/adapters/ci_secrets/exposure.py index 699156f..960a219 100644 --- a/ovk/adapters/ci_secrets/exposure.py +++ b/ovk/adapters/ci_secrets/exposure.py @@ -50,8 +50,7 @@ def find_ci_secrets_counterexamples(data: dict[str, Any]) -> list[dict[str, Any] { "summary": str(finding.get("summary") or kind), "failure_mode": FAILURE_MODE, - "workflow_id": ",".join(str(item) for item in finding.get("node_ids") or []) - or "trust-flow", + "workflow_id": ",".join(str(item) for item in finding.get("node_ids") or []) or "trust-flow", "triggers": [], "trust_context": trust_context, "trust_finding_kind": kind, diff --git a/ovk/adapters/ci_secrets/regression.py b/ovk/adapters/ci_secrets/regression.py index bc8a4e1..f2b3041 100644 --- a/ovk/adapters/ci_secrets/regression.py +++ b/ovk/adapters/ci_secrets/regression.py @@ -23,7 +23,10 @@ def render_ci_secrets_regression_test(counterexample: dict[str, Any], index: int def render_ci_secrets_regression_suite(counterexamples: list[dict[str, Any]]) -> str: if not counterexamples: return "# No CI secrets counterexamples were available.\n" - return "\n\n".join( - render_ci_secrets_regression_test(counterexample, index) - for index, counterexample in enumerate(counterexamples) - ) + "\n" + return ( + "\n\n".join( + render_ci_secrets_regression_test(counterexample, index) + for index, counterexample in enumerate(counterexamples) + ) + + "\n" + ) diff --git a/ovk/adapters/deployment/deterministic_adapter.py b/ovk/adapters/deployment/deterministic_adapter.py index 022d7c1..a1d0d6d 100644 --- a/ovk/adapters/deployment/deterministic_adapter.py +++ b/ovk/adapters/deployment/deterministic_adapter.py @@ -2,11 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.deployment.state_machine import find_skipped_approval_paths from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -24,7 +22,6 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus @@ -193,9 +190,7 @@ def normalize( assumptions=["Deterministic deployment state-machine evaluator."], limits=["Does not execute live deployment controllers."], counterexamples=list(raw.raw_result.get("counterexamples") or []), - generated_artifacts=[ - {"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False} - ], + generated_artifacts=[{"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False}], ) def explain(self, result: NormalizedBackendResult) -> HumanExplanation: diff --git a/ovk/adapters/deployment/diff_extract.py b/ovk/adapters/deployment/diff_extract.py index 9f31543..ab07c1c 100644 --- a/ovk/adapters/deployment/diff_extract.py +++ b/ovk/adapters/deployment/diff_extract.py @@ -35,8 +35,7 @@ def _normalize_deployment_payload(states: list[dict]) -> dict: required_states = [str(state["name"]) for state in states if state.get("requires_approval")] production_states = [state_names[-1]] if state_names else ["production"] state_metadata = { - str(state["name"]): {"requires_approval": bool(state.get("requires_approval"))} - for state in states + str(state["name"]): {"requires_approval": bool(state.get("requires_approval"))} for state in states } return { "initial_state": state_names[0] if state_names else "draft", diff --git a/ovk/adapters/deployment/regression.py b/ovk/adapters/deployment/regression.py index 46367a6..32aa8f5 100644 --- a/ovk/adapters/deployment/regression.py +++ b/ovk/adapters/deployment/regression.py @@ -22,7 +22,10 @@ def render_deployment_regression_test(counterexample: dict[str, Any], index: int def render_deployment_regression_suite(counterexamples: list[dict[str, Any]]) -> str: if not counterexamples: return "# No deployment state counterexamples were available.\n" - return "\n\n".join( - render_deployment_regression_test(counterexample, index) - for index, counterexample in enumerate(counterexamples) - ) + "\n" + return ( + "\n\n".join( + render_deployment_regression_test(counterexample, index) + for index, counterexample in enumerate(counterexamples) + ) + + "\n" + ) diff --git a/ovk/adapters/deployment/state_machine.py b/ovk/adapters/deployment/state_machine.py index cf73bf3..fa5a800 100644 --- a/ovk/adapters/deployment/state_machine.py +++ b/ovk/adapters/deployment/state_machine.py @@ -83,8 +83,7 @@ def find_skipped_approval_paths(data: dict[str, Any]) -> list[dict[str, Any]]: counterexamples.append( { "summary": ( - f"Production state {state} is reachable from {previous} " - "without a prior approval gate." + f"Production state {state} is reachable from {previous} without a prior approval gate." ), "failure_mode": FAILURE_MODE, "path": path, diff --git a/ovk/adapters/infra/graph.py b/ovk/adapters/infra/graph.py index 6ef9961..a9cfaad 100644 --- a/ovk/adapters/infra/graph.py +++ b/ovk/adapters/infra/graph.py @@ -40,9 +40,7 @@ def _adjacency(data: dict[str, Any]) -> dict[str, list[str]]: def _entrypoints(nodes: dict[str, dict[str, Any]]) -> list[str]: return [ - node_id - for node_id, node in nodes.items() - if node.get("external") is True or node.get("kind") == "external" + node_id for node_id, node in nodes.items() if node.get("external") is True or node.get("kind") == "external" ] diff --git a/ovk/adapters/infrastructure/deterministic_adapter.py b/ovk/adapters/infrastructure/deterministic_adapter.py index fda4c4c..1ace646 100644 --- a/ovk/adapters/infrastructure/deterministic_adapter.py +++ b/ovk/adapters/infrastructure/deterministic_adapter.py @@ -2,12 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.infra.exposure import find_exposure_counterexamples -from ovk.adapters.infra.validation import validate_infra_input from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -25,7 +22,6 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus @@ -194,9 +190,7 @@ def normalize( assumptions=["Deterministic infrastructure exposure evaluator."], limits=["Does not claim Terraform plan or Kubernetes API execution."], counterexamples=list(raw.raw_result.get("counterexamples") or []), - generated_artifacts=[ - {"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False} - ], + generated_artifacts=[{"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False}], ) def explain(self, result: NormalizedBackendResult) -> HumanExplanation: diff --git a/ovk/adapters/lane/_base.py b/ovk/adapters/lane/_base.py index 2f08d6b..2d52d14 100644 --- a/ovk/adapters/lane/_base.py +++ b/ovk/adapters/lane/_base.py @@ -186,9 +186,7 @@ def compile( }, expected_guarantee=self.guarantee_type, ) - return provisional.model_copy( - update={"backend_obligation_id": compute_backend_obligation_id(provisional)} - ) + return provisional.model_copy(update={"backend_obligation_id": compute_backend_obligation_id(provisional)}) def fingerprint(self, backend_obligation: BackendObligation) -> BackendEnvironmentFingerprint: env_payload = { diff --git a/ovk/adapters/opa/infra_exposure.py b/ovk/adapters/opa/infra_exposure.py index 5c4a2f2..5279b24 100644 --- a/ovk/adapters/opa/infra_exposure.py +++ b/ovk/adapters/opa/infra_exposure.py @@ -9,7 +9,8 @@ from ovk.core.models import VerificationEvidence -INFRA_EXPOSURE_REGO = r''' +INFRA_EXPOSURE_REGO = ( + r""" package ovk.infra_exposure violation[msg] { @@ -25,7 +26,9 @@ resource.public_exposure == true msg := sprintf("sensitive resource publicly exposed: %s", [resource.resource_id]) } -'''.strip() + "\n" +""".strip() + + "\n" +) def _deterministic_infra_opa(data: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]: @@ -33,7 +36,9 @@ def _deterministic_infra_opa(data: dict[str, Any]) -> tuple[str, list[dict[str, if counterexamples: return "fail", counterexamples if not data.get("resources"): - return "unknown", [{"summary": "resources must be a non-empty list", "failure_mode": "infrastructure_abstraction_invalid"}] + return "unknown", [ + {"summary": "resources must be a non-empty list", "failure_mode": "infrastructure_abstraction_invalid"} + ] return "pass", [] diff --git a/ovk/adapters/opa/policy_assets.py b/ovk/adapters/opa/policy_assets.py index 3771bc8..d696d9c 100644 --- a/ovk/adapters/opa/policy_assets.py +++ b/ovk/adapters/opa/policy_assets.py @@ -10,7 +10,8 @@ from pathlib import Path -SELF_PROTECTION_REGO = r''' +SELF_PROTECTION_REGO = ( + r""" package ovk.self_protection after_has_gate(gate) { @@ -38,7 +39,9 @@ input.after.workflow_permissions.actions == "write" msg := "workflow actions permission escalated to write" } -'''.strip() + "\n" +""".strip() + + "\n" +) def write_self_protection_rego(path: Path) -> None: diff --git a/ovk/adapters/opa/self_protection.py b/ovk/adapters/opa/self_protection.py index 3f00c38..0239605 100644 --- a/ovk/adapters/opa/self_protection.py +++ b/ovk/adapters/opa/self_protection.py @@ -133,7 +133,9 @@ def find_self_protection_violations(data: dict[str, Any]) -> list[SelfProtection permissions_after = _phase(data, "after").get("workflow_permissions", {}) permissions_before = _phase(data, "before").get("workflow_permissions", {}) if isinstance(permissions_after, dict): - before_actions = str(permissions_before.get("actions", "read")) if isinstance(permissions_before, dict) else "read" + before_actions = ( + str(permissions_before.get("actions", "read")) if isinstance(permissions_before, dict) else "read" + ) after_actions = str(permissions_after.get("actions", "read")) if before_actions != "write" and after_actions == "write": violations.append( diff --git a/ovk/adapters/self_protection/deterministic_adapter.py b/ovk/adapters/self_protection/deterministic_adapter.py index 27674c0..b5359d1 100644 --- a/ovk/adapters/self_protection/deterministic_adapter.py +++ b/ovk/adapters/self_protection/deterministic_adapter.py @@ -2,14 +2,9 @@ from __future__ import annotations -import time from datetime import datetime, timezone from typing import Any -from ovk.adapters.opa.self_protection import ( - find_self_protection_unknowns, - find_self_protection_violations, -) from ovk.core.bundle import content_digest from ovk.core.execution_models import ( BackendCapabilityAssessment, @@ -27,7 +22,6 @@ VerificationObligation, compute_backend_obligation_id, compute_payload_digest, - compute_raw_execution_digests, ) from ovk.core.execution_budget import BackendWorker from ovk.core.models import VerificationStatus @@ -109,11 +103,7 @@ def can_handle( estimated_memory_mb=64, reasons=["excluded by execution budget"], ) - trusted_meta = all( - item.trusted - for item in obligation.materials - if item.kind == "branch_protection" - ) + trusted_meta = all(item.trusted for item in obligation.materials if item.kind == "branch_protection") materials_ok = bool(obligation.materials) coverage_ok = obligation.coverage.status in {"complete", "partial", "unknown"} score = 0.8 if trusted_meta else 0.55 @@ -199,9 +189,7 @@ def normalize( assumptions=["Deterministic self-protection evaluator."], limits=["OPA native execution is a separate backend."], counterexamples=list(raw.raw_result.get("counterexamples") or []), - generated_artifacts=[ - {"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False} - ], + generated_artifacts=[{"kind": "backend_provenance", "backend": self.backend_id, "native_execution": False}], ) def explain(self, result: NormalizedBackendResult) -> HumanExplanation: diff --git a/ovk/adapters/self_protection/opa_adapter.py b/ovk/adapters/self_protection/opa_adapter.py index b7aab3d..3fe8895 100644 --- a/ovk/adapters/self_protection/opa_adapter.py +++ b/ovk/adapters/self_protection/opa_adapter.py @@ -242,9 +242,7 @@ def run( status = str(result.get("status", "unknown")) violations = result.get("violations") or [] counterexamples = [ - {"summary": str(item), "failure_mode": "opa_policy_violation"} - if not isinstance(item, dict) - else item + {"summary": str(item), "failure_mode": "opa_policy_violation"} if not isinstance(item, dict) else item for item in violations ] termination = "completed" @@ -313,7 +311,9 @@ def explain(self, result: NormalizedBackendResult) -> HumanExplanation: failure_mode=str(result.counterexamples[0].get("failure_mode", "opa_policy_violation")), ) if result.status == VerificationStatus.PASS: - return HumanExplanation(summary="OPA reported no self-protection violations.", repair_hint="No repair required.") + return HumanExplanation( + summary="OPA reported no self-protection violations.", repair_hint="No repair required." + ) return HumanExplanation( summary=f"OPA-native backend returned {result.status.value}.", repair_hint="Install opa or select an explicitly accepted fallback backend via policy.", diff --git a/ovk/adapters/z3/obligation.py b/ovk/adapters/z3/obligation.py index 13d6e1a..043f13c 100644 --- a/ovk/adapters/z3/obligation.py +++ b/ovk/adapters/z3/obligation.py @@ -82,10 +82,7 @@ def obligation_to_dict(obligation: AuthorizationObligation) -> dict[str, Any]: "path": route.path, "admin_only_before": route.admin_only_before, "admin_only_after": route.admin_only_after, - "reachable_after": [ - {"role": witness.role, "via": witness.via} - for witness in route.reachable_after - ], + "reachable_after": [{"role": witness.role, "via": witness.via} for witness in route.reachable_after], } for route in obligation.routes ], diff --git a/ovk/adapters/z3/regression.py b/ovk/adapters/z3/regression.py index 28cf5de..24deda2 100644 --- a/ovk/adapters/z3/regression.py +++ b/ovk/adapters/z3/regression.py @@ -26,5 +26,9 @@ def render_authorization_regression_suite(counterexamples: list[dict[str, Any]]) tests = [] for index, counterexample in enumerate(counterexamples): test_body = render_authorization_regression_test(counterexample) - tests.append(test_body.replace("test_non_admin_cannot_reach_admin_route", f"test_non_admin_cannot_reach_admin_route_{index}")) + tests.append( + test_body.replace( + "test_non_admin_cannot_reach_admin_route", f"test_non_admin_cannot_reach_admin_route_{index}" + ) + ) return "\n\n".join(tests) + "\n" diff --git a/ovk/adapters/z3/smt_plan.py b/ovk/adapters/z3/smt_plan.py index b1aeb01..98a15b2 100644 --- a/ovk/adapters/z3/smt_plan.py +++ b/ovk/adapters/z3/smt_plan.py @@ -56,8 +56,5 @@ def smt_plan_to_dict(plan: SmtPlan) -> dict: return { "obligation_id": plan.obligation_id, "query_polarity": plan.query_polarity, - "clauses": [ - {"name": clause.name, "expression": clause.expression} - for clause in plan.clauses - ], + "clauses": [{"name": clause.name, "expression": clause.expression} for clause in plan.clauses], } diff --git a/ovk/cli.py b/ovk/cli.py index 4c91747..8122f1b 100644 --- a/ovk/cli.py +++ b/ovk/cli.py @@ -298,7 +298,9 @@ def release_bundle( base_sha: Optional[str] = typer.Option(None), ) -> None: """Write a complete verifiable release bundle for a lane.""" - evidence = _evaluate_lane(lane, input_json, input_format=input_format, policy=policy, repo=repo, head_sha=head_sha, base_sha=base_sha) + evidence = _evaluate_lane( + lane, input_json, input_format=input_format, policy=policy, repo=repo, head_sha=head_sha, base_sha=base_sha + ) bundle = make_bundle([evidence]) write_release_bundle(bundle, ReleaseBundlePaths(root=output_dir)) failures = verify_release_bundle(output_dir) @@ -324,7 +326,9 @@ def _evaluate_lane( return evaluate_validated_authorization_path(data, repo=repo, head_sha=head_sha, base_sha=base_sha) if lane == "infrastructure": normalized = normalize_infra_input(data, input_format) - return evaluate_infra_exposure(normalized, repo=repo, head_sha=head_sha, base_sha=base_sha, policy=load_policy(policy)) + return evaluate_infra_exposure( + normalized, repo=repo, head_sha=head_sha, base_sha=base_sha, policy=load_policy(policy) + ) if lane == "self_protection": return evaluate_self_protection(data, repo=repo, head_sha=head_sha, base_sha=base_sha) if lane == "ci_secrets": @@ -409,8 +413,12 @@ def auth_obligation( evidence_output: Path = typer.Option(Path("ovk-auth-evidence.json"), help="Evidence bundle output path."), markdown_output: Path = typer.Option(Path("ovk-auth-comment.md"), help="Markdown output path."), attestation_output: Path = typer.Option(Path("ovk-auth-attestation.json"), help="Attestation output path."), - manifest_output: Path = typer.Option(Path("ovk-auth-artifact-manifest.json"), help="Artifact manifest output path."), - quality_output: Path = typer.Option(Path("ovk-auth-evidence-quality.json"), help="Evidence quality report output path."), + manifest_output: Path = typer.Option( + Path("ovk-auth-artifact-manifest.json"), help="Artifact manifest output path." + ), + quality_output: Path = typer.Option( + Path("ovk-auth-evidence-quality.json"), help="Evidence quality report output path." + ), advisory: bool = typer.Option(False, help="Write outputs and exit 0."), ) -> None: """Run the validated authorization obligation path.""" @@ -439,8 +447,12 @@ def infra_exposure( evidence_output: Path = typer.Option(Path("ovk-infra-evidence.json"), help="Evidence bundle output path."), markdown_output: Path = typer.Option(Path("ovk-infra-comment.md"), help="Markdown output path."), attestation_output: Path = typer.Option(Path("ovk-infra-attestation.json"), help="Attestation output path."), - manifest_output: Path = typer.Option(Path("ovk-infra-artifact-manifest.json"), help="Artifact manifest output path."), - quality_output: Path = typer.Option(Path("ovk-infra-evidence-quality.json"), help="Evidence quality report output path."), + manifest_output: Path = typer.Option( + Path("ovk-infra-artifact-manifest.json"), help="Artifact manifest output path." + ), + quality_output: Path = typer.Option( + Path("ovk-infra-evidence-quality.json"), help="Evidence quality report output path." + ), advisory: bool = typer.Option(False, help="Write outputs and exit 0."), ) -> None: """Run the infrastructure exposure path.""" @@ -473,8 +485,12 @@ def ci_secrets( evidence_output: Path = typer.Option(Path("ovk-ci-secrets-evidence.json"), help="Evidence bundle output path."), markdown_output: Path = typer.Option(Path("ovk-ci-secrets-comment.md"), help="Markdown output path."), attestation_output: Path = typer.Option(Path("ovk-ci-secrets-attestation.json"), help="Attestation output path."), - manifest_output: Path = typer.Option(Path("ovk-ci-secrets-artifact-manifest.json"), help="Artifact manifest output path."), - quality_output: Path = typer.Option(Path("ovk-ci-secrets-evidence-quality.json"), help="Evidence quality report output path."), + manifest_output: Path = typer.Option( + Path("ovk-ci-secrets-artifact-manifest.json"), help="Artifact manifest output path." + ), + quality_output: Path = typer.Option( + Path("ovk-ci-secrets-evidence-quality.json"), help="Evidence quality report output path." + ), advisory: bool = typer.Option(False, help="Write outputs and exit 0."), ) -> None: """Run the CI secrets exposure path.""" @@ -500,12 +516,18 @@ def deployment_state( evidence_output: Path = typer.Option(Path("ovk-deployment-evidence.json"), help="Evidence bundle output path."), markdown_output: Path = typer.Option(Path("ovk-deployment-comment.md"), help="Markdown output path."), attestation_output: Path = typer.Option(Path("ovk-deployment-attestation.json"), help="Attestation output path."), - manifest_output: Path = typer.Option(Path("ovk-deployment-artifact-manifest.json"), help="Artifact manifest output path."), - quality_output: Path = typer.Option(Path("ovk-deployment-evidence-quality.json"), help="Evidence quality report output path."), + manifest_output: Path = typer.Option( + Path("ovk-deployment-artifact-manifest.json"), help="Artifact manifest output path." + ), + quality_output: Path = typer.Option( + Path("ovk-deployment-evidence-quality.json"), help="Evidence quality report output path." + ), advisory: bool = typer.Option(False, help="Write outputs and exit 0."), ) -> None: """Run the deployment approval state machine path.""" - evidence = evaluate_approval_state_machine(read_json_file(input_json), repo=repo, head_sha=head_sha, base_sha=base_sha) + evidence = evaluate_approval_state_machine( + read_json_file(input_json), repo=repo, head_sha=head_sha, base_sha=base_sha + ) _finish_lane( make_bundle([evidence]), label="deployment", @@ -725,7 +747,9 @@ def run_cmd( @app.command("generate-test") def generate_test( evidence_bundle: Path = typer.Option(..., "--evidence", help="Evidence bundle JSON."), - output_dir: Path = typer.Option(Path(".verification/generated_tests"), help="Regression artifact output directory."), + output_dir: Path = typer.Option( + Path(".verification/generated_tests"), help="Regression artifact output directory." + ), ) -> None: """Generate regression artifacts from bundle counterexamples.""" bundle = EvidenceBundle.model_validate(read_json_file(evidence_bundle)) @@ -740,11 +764,7 @@ def repair_suggest( ) -> None: """Emit machine-readable repair hints from bundle counterexamples.""" bundle = EvidenceBundle.model_validate(read_json_file(evidence_bundle)) - counterexamples = [ - counterexample - for evidence in bundle.evidence - for counterexample in evidence.counterexamples - ] + counterexamples = [counterexample for evidence in bundle.evidence for counterexample in evidence.counterexamples] if not counterexamples: typer.echo(json.dumps({"repair_hints": [], "blocked": bundle.decision.get("merge_recommendation") == "block"})) return diff --git a/ovk/compilers/authorization/corpus.py b/ovk/compilers/authorization/corpus.py index cf70b85..973191b 100644 --- a/ovk/compilers/authorization/corpus.py +++ b/ovk/compilers/authorization/corpus.py @@ -187,11 +187,7 @@ def build_corpus(*, meet_targets: bool = True) -> list[CorpusCase]: def classify_case(case: CorpusCase) -> Category: - compiler = ( - FastApiAuthorizationCompiler() - if case.framework == "fastapi" - else ExpressAuthorizationCompiler() - ) + compiler = FastApiAuthorizationCompiler() if case.framework == "fastapi" else ExpressAuthorizationCompiler() materials = materials_from_pair( path=case.path, base_source=case.base if case.base else None, diff --git a/ovk/compilers/authorization/express.py b/ovk/compilers/authorization/express.py index 417c38e..07c6c93 100644 --- a/ovk/compilers/authorization/express.py +++ b/ovk/compilers/authorization/express.py @@ -76,7 +76,9 @@ def _index(self, files: dict[str, str]) -> dict: for path, source in sorted(files.items()): for match in _IMPORT.finditer(source): - name = match.group("named") or match.group("default") or match.group("imnamed") or match.group("imdefault") + name = ( + match.group("named") or match.group("default") or match.group("imnamed") or match.group("imdefault") + ) mod = match.group("mod") or match.group("immod") if not name: continue diff --git a/ovk/compilers/authorization/fastapi_ast.py b/ovk/compilers/authorization/fastapi_ast.py index c45f615..64ef41b 100644 --- a/ovk/compilers/authorization/fastapi_ast.py +++ b/ovk/compilers/authorization/fastapi_ast.py @@ -123,7 +123,9 @@ def _index(self, files: dict[str, str]) -> dict[str, Any]: for node in tree.body: if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - roles = ["admin"] if looks_admin_protected(ast.get_source_segment(source, node) or node.name) else [] + roles = ( + ["admin"] if looks_admin_protected(ast.get_source_segment(source, node) or node.name) else [] + ) dependencies[node.name] = AuthDependency( name=node.name, kind="dependency", diff --git a/ovk/compilers/cbmc/traceability.py b/ovk/compilers/cbmc/traceability.py index f7ffe01..c8ef263 100644 --- a/ovk/compilers/cbmc/traceability.py +++ b/ovk/compilers/cbmc/traceability.py @@ -36,7 +36,5 @@ def validate_project_traceability(project: CbmcProject) -> list[str]: if record.get(field) in (None, [], ""): failures.append(f"{harness.harness_id}: missing {field}") if project.guarantee_type == "bounded_project_model_check" and not harness.includes_project_code: - failures.append( - f"{harness.harness_id}: bounded_project_model_check requires includes_project_code" - ) + failures.append(f"{harness.harness_id}: bounded_project_model_check requires includes_project_code") return failures diff --git a/ovk/compilers/deployment/ir.py b/ovk/compilers/deployment/ir.py index 88aa37d..9347bb3 100644 --- a/ovk/compilers/deployment/ir.py +++ b/ovk/compilers/deployment/ir.py @@ -37,10 +37,7 @@ def to_lane_input(self) -> dict[str, Any]: return { "initial_state": self.initial_state, "states": [item.name for item in self.states], - "transitions": [ - {"from": item.source, "to": item.target, "label": item.label} - for item in self.transitions - ], + "transitions": [{"from": item.source, "to": item.target, "label": item.label} for item in self.transitions], "required_states": list(self.required_states), "production_states": list(self.production_states), "warnings": list(self.warnings), diff --git a/ovk/compilers/github_actions/composite_actions.py b/ovk/compilers/github_actions/composite_actions.py index 83ea06b..de3e447 100644 --- a/ovk/compilers/github_actions/composite_actions.py +++ b/ovk/compilers/github_actions/composite_actions.py @@ -31,9 +31,7 @@ def expand_composite_action( nodes: list[TrustNode] = [ TrustNode(node_id=f"action:{action_path.as_posix()}", kind="composite_action", trust="unknown") ] - edges: list[TrustEdge] = [ - TrustEdge(source=step_node_id, target=nodes[0].node_id, kind="uses_composite") - ] + edges: list[TrustEdge] = [TrustEdge(source=step_node_id, target=nodes[0].node_id, kind="uses_composite")] secrets: list[SecretUse] = [] for index, step in enumerate(runs.get("steps") or []): if not isinstance(step, dict): diff --git a/ovk/compilers/github_actions/permissions.py b/ovk/compilers/github_actions/permissions.py index 9b7adab..c6ed039 100644 --- a/ovk/compilers/github_actions/permissions.py +++ b/ovk/compilers/github_actions/permissions.py @@ -33,6 +33,5 @@ def _from_block(block: Any, *, job_id: str | None) -> list[PermissionGrant]: if not isinstance(block, dict): return [] return [ - PermissionGrant(scope=str(scope), level=str(level), job_id=job_id) - for scope, level in sorted(block.items()) + PermissionGrant(scope=str(scope), level=str(level), job_id=job_id) for scope, level in sorted(block.items()) ] diff --git a/ovk/compilers/github_actions/reusable_workflows.py b/ovk/compilers/github_actions/reusable_workflows.py index 2f0e9d0..1c156de 100644 --- a/ovk/compilers/github_actions/reusable_workflows.py +++ b/ovk/compilers/github_actions/reusable_workflows.py @@ -105,9 +105,7 @@ def resolve_local_reusable( child = load_workflow_file(local_path) child["_ovk_path"] = str(local_path.resolve()) loaded.append(child) - nested, nested_findings = resolve_local_reusable( - child, repo_root=repo_root, visiting=set(visiting) - ) + nested, nested_findings = resolve_local_reusable(child, repo_root=repo_root, visiting=set(visiting)) loaded.extend(nested) findings.extend(nested_findings) if path: diff --git a/ovk/compilers/github_actions/secrets.py b/ovk/compilers/github_actions/secrets.py index daf47c1..c55a78c 100644 --- a/ovk/compilers/github_actions/secrets.py +++ b/ovk/compilers/github_actions/secrets.py @@ -49,10 +49,7 @@ def _scan(value: Any, *, job_id: str | None, step_id: str | None) -> list[Secret if value is None: return [] if isinstance(value, str): - return [ - SecretUse(name=name, job_id=job_id, step_id=step_id, expression=value) - for name in secret_names(value) - ] + return [SecretUse(name=name, job_id=job_id, step_id=step_id, expression=value) for name in secret_names(value)] if isinstance(value, dict): found: list[SecretUse] = [] for nested in value.values(): diff --git a/ovk/compilers/github_actions/trust_flow.py b/ovk/compilers/github_actions/trust_flow.py index d84d698..bb9a854 100644 --- a/ovk/compilers/github_actions/trust_flow.py +++ b/ovk/compilers/github_actions/trust_flow.py @@ -51,9 +51,7 @@ def compile_workflow_trust( permissions = extract_permissions(workflow) secrets = extract_secrets(workflow) - write_token = has_write_token(permissions) or any( - references_github_token(item.expression) for item in secrets - ) + write_token = has_write_token(permissions) or any(references_github_token(item.expression) for item in secrets) jobs = workflow.get("jobs") if isinstance(workflow.get("jobs"), dict) else {} for job_id, job in sorted(jobs.items()): @@ -102,8 +100,10 @@ def compile_workflow_trust( step_node_id = f"job:{job_id}:step:{step_id}" run = str(step.get("run") or "") uses = str(step.get("uses") or "") - untrusted_code = untrusted_trigger or contains_untrusted_context(run) or contains_untrusted_context( - str(step.get("with") or "") + untrusted_code = ( + untrusted_trigger + or contains_untrusted_context(run) + or contains_untrusted_context(str(step.get("with") or "")) ) step_node = TrustNode( node_id=step_node_id, @@ -174,9 +174,7 @@ def compile_workflow_trust( # resolve_local_reusable already walked the graph with cycle prevention. for child in reusable: child_path = str(child.get("_ovk_path") or "reusable.yml") - nodes.append( - TrustNode(node_id=f"workflow:{child_path}", kind="reusable_workflow_doc", trust="unknown") - ) + nodes.append(TrustNode(node_id=f"workflow:{child_path}", kind="reusable_workflow_doc", trust="unknown")) child_secrets = extract_secrets(child) child_permissions = extract_permissions(child) secrets.extend(child_secrets) diff --git a/ovk/compilers/infrastructure/exposure_graph.py b/ovk/compilers/infrastructure/exposure_graph.py index b34d62b..aedfe7c 100644 --- a/ovk/compilers/infrastructure/exposure_graph.py +++ b/ovk/compilers/infrastructure/exposure_graph.py @@ -95,9 +95,7 @@ def apply_concrete_exposure(resources: list[InfraResourceIR], paths: list[Exposu updated: list[InfraResourceIR] = [] for resource in resources: path_labels = [ - " -> ".join(path.nodes) - for path in paths - if path.is_concrete and path.nodes[-1] == resource.resource_id + " -> ".join(path.nodes) for path in paths if path.is_concrete and path.nodes[-1] == resource.resource_id ] updated.append( resource.model_copy( diff --git a/ovk/compilers/infrastructure/kubernetes.py b/ovk/compilers/infrastructure/kubernetes.py index e12a2c5..3306710 100644 --- a/ovk/compilers/infrastructure/kubernetes.py +++ b/ovk/compilers/infrastructure/kubernetes.py @@ -128,8 +128,7 @@ def compile_kubernetes_objects(objects: list[dict[str, Any]] | dict[str, Any]) - attributes["pod_security_labels"] = { key: value for key, value in _meta(obj).get("labels", {}).items() - if isinstance(_meta(obj).get("labels"), dict) - and str(key).startswith("pod-security.kubernetes.io/") + if isinstance(_meta(obj).get("labels"), dict) and str(key).startswith("pod-security.kubernetes.io/") } else: unsupported.append(f"{resource_id}:unsupported_kind:{kind}") @@ -210,11 +209,7 @@ def _controller_reachability_edges( if not isinstance(selector, dict): continue for controller_id, controller in controllers: - template = ( - controller.get("spec", {}).get("template") - if isinstance(controller.get("spec"), dict) - else None - ) + template = controller.get("spec", {}).get("template") if isinstance(controller.get("spec"), dict) else None labels = _labels(template) if isinstance(template, dict) else {} if not _selector_match(selector, labels): continue diff --git a/ovk/compilers/infrastructure/reachability.py b/ovk/compilers/infrastructure/reachability.py index e9b32fd..5120c60 100644 --- a/ovk/compilers/infrastructure/reachability.py +++ b/ovk/compilers/infrastructure/reachability.py @@ -27,7 +27,11 @@ def evaluate_eligibility(ir: InfrastructureIR) -> InfrastructureIR: reasons.append(f"{resource.resource_id} marked public without concrete path") # Sensitive + public without concrete path is always review. for resource in ir.resources: - if is_sensitive(resource.sensitivity) and resource.public_exposure and not _has_concrete(resource, ir.public_paths): + if ( + is_sensitive(resource.sensitivity) + and resource.public_exposure + and not _has_concrete(resource, ir.public_paths) + ): reasons.append(f"sensitive resource {resource.resource_id} lacks concrete exposure path") eligibility: Eligibility = "strict" if not reasons and ir.resources else "review" @@ -46,7 +50,5 @@ def sensitive_public_violations(ir: InfrastructureIR) -> list[InfraResourceIR]: return [ resource for resource in ir.resources - if is_sensitive(resource.sensitivity) - and resource.public_exposure - and _has_concrete(resource, ir.public_paths) + if is_sensitive(resource.sensitivity) and resource.public_exposure and _has_concrete(resource, ir.public_paths) ] diff --git a/ovk/compilers/infrastructure/terraform_plan.py b/ovk/compilers/infrastructure/terraform_plan.py index 69795c7..94bf1b0 100644 --- a/ovk/compilers/infrastructure/terraform_plan.py +++ b/ovk/compilers/infrastructure/terraform_plan.py @@ -117,9 +117,7 @@ def compile_terraform_plan(plan: dict[str, Any]) -> InfrastructureIR: nested = expand_planned_values_recursively(plan, warnings) if nested: existing = { - str(item.get("address")) - for item in resource_changes - if isinstance(item, dict) and item.get("address") + str(item.get("address")) for item in resource_changes if isinstance(item, dict) and item.get("address") } added = 0 for item in nested: diff --git a/ovk/core/adapter_runtime.py b/ovk/core/adapter_runtime.py index e7a6747..9e72b7a 100644 --- a/ovk/core/adapter_runtime.py +++ b/ovk/core/adapter_runtime.py @@ -8,12 +8,9 @@ """ - - from __future__ import annotations - from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -21,7 +18,6 @@ from typing import Any, Mapping - from ovk.adapters.authorization import build_authorization_registry from ovk.adapters.ci_secrets import build_ci_secrets_registry @@ -51,31 +47,20 @@ from ovk.core.policy_config import resolve_routing_config, routing_enforced_for_lane from ovk.core.result_cache import ( - ControlPlaneResultCache, - HardenedResultCache, - cache_key, - get_cached_evidence, - store_cached_evidence, - ) -from ovk.core.router import RoutingConfig, route_obligation, routing_config_from_policy, routing_decision_to_legacy_dict +from ovk.core.router import route_obligation, routing_decision_to_legacy_dict from ovk.core.routing_pipeline import ( - AuthoritativeRoutingPlan, - ensure_authoritative_routing, - intent_id_for_obligation, - require_routing_decision, - ) from ovk.core.self_protection_compiler import resolve_metadata_trusted @@ -83,85 +68,50 @@ from ovk.core.shadow_obligation import build_shadow_obligation - - - def _control_plane(*, cache_dir: Path | None = None) -> BackendControlPlane: - """Build an enforced/shadow control plane with hardened cache + worker.""" hardened = HardenedResultCache(cache_dir / "control-plane") if cache_dir is not None else HardenedResultCache() return BackendControlPlane( - cache=ControlPlaneResultCache(hardened), - worker=LocalSubprocessWorker(), - use_hardened_cache=True, - ) - - - LANE_TO_INTENT = { - "self_protection": "agent-cannot-disable-own-ci-gate", - "authorization": "no-admin-route-bypass", - "infrastructure": "no-public-sensitive-resource", - "ci_secrets": "no-secrets-in-untrusted-context", - "deployment": "no-skipped-approval-state", - } - _LANE_REGISTRY_BUILDERS = { - "authorization": build_authorization_registry, - "self_protection": build_self_protection_registry, - "infrastructure": build_infrastructure_registry, - "ci_secrets": build_ci_secrets_registry, - "deployment": build_deployment_registry, - } - - - def _routing_metadata( - routing: RoutingDecision | Mapping[str, Any] | None, - *, - intent_id: str, - routing_enforced: bool, - ) -> dict[str, Any] | None: if routing is None: - return None if isinstance(routing, RoutingDecision): - payload = routing_decision_to_legacy_dict(routing, intent_id=intent_id) else: - payload = dict(routing) payload["routing_enforced"] = routing_enforced @@ -169,51 +119,34 @@ def _routing_metadata( return payload - - - def _attach_execution_metadata( - evidence: VerificationEvidence, - *, - lane: str, - data: dict[str, Any], - routing: RoutingDecision | Mapping[str, Any] | None, - shadow_comparison: dict[str, Any] | None = None, - intent_id: str | None = None, - job_id: str | None = None, - input_format: str | None = None, - routing_enforced: bool = False, - ) -> VerificationEvidence: - """Record routing, input digest, and obligation-scoped evidence identity.""" - resolved_intent = intent_id or str((routing or {}).get("intent_id") if isinstance(routing, dict) else None) or LANE_TO_INTENT.get(lane, lane) + resolved_intent = ( + intent_id + or str((routing or {}).get("intent_id") if isinstance(routing, dict) else None) + or LANE_TO_INTENT.get(lane, lane) + ) resolved_format = input_format or "infra" identity = { - "intent_id": resolved_intent, - "lane": lane, - "input": data, - "input_format": resolved_format, - "job_id": job_id, - } input_digest = content_digest({"lane": lane, "input": data}) @@ -227,51 +160,30 @@ def _attach_execution_metadata( metadata = _routing_metadata(routing, intent_id=resolved_intent, routing_enforced=routing_enforced) if metadata is not None: - artifacts.append( - { - "kind": "backend_routing", - "intent_id": metadata.get("intent_id", resolved_intent), - "selected": metadata.get("selected", []), - "rejected": metadata.get("rejected", []), - "routing_id": metadata.get("routing_id") or evidence.routing_id, - "routing_enforced": routing_enforced, - "executed_backends": [claim.backend for claim in evidence.backend_claims], - } - ) if shadow_comparison is not None: - artifacts.append(shadow_comparison) return evidence.model_copy( - update={ - "evidence_id": f"{evidence.evidence_id}-{evidence_suffix}", - "generated_artifacts": artifacts, - "routing_id": evidence.routing_id or (metadata or {}).get("routing_id"), - } - ) - - - def _legacy_status_and_recommendation(evidence: VerificationEvidence) -> tuple[str, str]: status = evidence.backend_claims[0].status.value if evidence.backend_claims else "unknown" @@ -281,63 +193,37 @@ def _legacy_status_and_recommendation(evidence: VerificationEvidence) -> tuple[s return status, recommendation - - - def _run_shadow_path( - *, - lane: str, - data: dict[str, Any], - repo: str, - head_sha: str, - base_sha: str | None, - intent_id: str, - policy: dict[str, Any] | None, - ) -> dict[str, Any] | None: - """Execute the typed control plane for comparison; never raises to legacy.""" try: - registry = build_default_lane_registry() obligation = build_shadow_obligation( - lane=lane, - data=data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - intent_id=intent_id, - ) budget = execution_budget_from_policy(policy) context = ExecutionContext( - subject=obligation.subject, - budget=budget, - policy_digest=obligation.policy_digest, - metadata={"shadow": True}, - ) routing = route_obligation(obligation, registry, context=context, policy=policy) @@ -345,55 +231,33 @@ def _run_shadow_path( record = _control_plane().execute(obligation, routing, registry=registry) return { - "record": record, - "routing": routing, - } except Exception as exc: # noqa: BLE001 - shadow must not affect legacy authority - return { - "error": { - "category": type(exc).__name__, - "message": str(exc), - } - } - - - def _run_enforced_with_routing( - *, - lane: str, - data: dict[str, Any], - routing: RoutingDecision, - typed_obligation: VerificationObligation, - policy: dict[str, Any] | None, - schema_version: str, - ) -> VerificationEvidence: - """Execute one enforced lane using a pre-computed immutable routing decision.""" registry_builder = _LANE_REGISTRY_BUILDERS.get(lane) if registry_builder is None: - raise RuntimeError(f"no enforced registry for lane {lane!r}") registry = registry_builder() @@ -401,77 +265,44 @@ def _run_enforced_with_routing( record = _control_plane().execute(typed_obligation, routing, registry=registry) evidence = execution_record_to_evidence( - record, - author_type=str((data.get("actor") or {}).get("type", data.get("author_type", "unknown"))), - agent=str((data.get("actor") or {}).get("id", data.get("agent", "unknown"))), - task=str(data.get("task", "unknown")), - routing_enforced=True, - schema_version=schema_version, - ) if lane == "self_protection": - metadata_trusted = resolve_metadata_trusted(policy) if not metadata_trusted and evidence.decision.get("merge_recommendation") == "allow": - evidence = evidence.model_copy( - update={ - "decision": { - **evidence.decision, - "merge_recommendation": "require_human_review", - "human_review_required": True, - "reason": "untrusted metadata cannot authorize allow under enforcement", - "fallback_accepted": False, - } - } - ) return evidence - - - def _evaluate_obligation( - obligation: dict[str, Any], - *, - routing_plan: AuthoritativeRoutingPlan, - repo: str, - head_sha: str, - base_sha: str | None, - cache_dir: Path | None, - use_cache: bool, - policy: dict[str, Any] | None = None, - evidence_schema_version: str = "ovk.evidence.v3", - ) -> VerificationEvidence: lane = str(obligation["lane"]) @@ -483,169 +314,99 @@ def _evaluate_obligation( input_format = str(obligation.get("input_format", "infra")) key = cache_key( - lane, - data, - subject={"repo": repo, "head_sha": head_sha, **({"base_sha": base_sha} if base_sha else {})}, - execution_fingerprint={"intent_id": intent_id, "input_format": input_format}, - ) precomputed_routing = routing_plan.routing_by_intent.get(intent_id) precomputed_typed = routing_plan.typed_obligations.get(intent_id) - - if use_cache and cache_dir is not None: - cached = get_cached_evidence(cache_dir, key) if cached is not None: - evidence = VerificationEvidence.model_validate(cached) return _attach_execution_metadata( - evidence, - lane=lane, - data=data, - routing=precomputed_routing, - intent_id=intent_id, - job_id=obligation.get("job_id"), - input_format=input_format, - routing_enforced=routing_enforced_for_lane(policy, lane), - ) - - if routing_enforced_for_lane(policy, lane): - routing_decision = require_routing_decision( - precomputed_routing, - intent_id=intent_id, - lane=lane, - policy=policy, - ) if precomputed_typed is None: - raise RuntimeError(f"missing typed obligation for enforced intent {intent_id!r}") evidence = _run_enforced_with_routing( - lane=lane, - data=data, - routing=routing_decision, - typed_obligation=precomputed_typed, - policy=policy, - schema_version=evidence_schema_version, - ) if use_cache and cache_dir is not None: - store_cached_evidence(cache_dir, key, evidence.model_dump(mode="json")) return _attach_execution_metadata( - evidence, - lane=lane, - data=data, - routing=routing_decision, - intent_id=intent_id, - job_id=obligation.get("job_id"), - input_format=input_format, - routing_enforced=True, - ) - - evidence = evaluate_lane( - lane, - data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - input_format=input_format, - policy_path=Path(obligation["policy_path"]) if obligation.get("policy_path") else None, - ) - - shadow_comparison: dict[str, Any] | None = None routing_config = resolve_routing_config(policy) if routing_config.mode in {"shadow", "enforced"} and lane in _LANE_REGISTRY_BUILDERS: - shadow = _run_shadow_path( - lane=lane, - data=data, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - intent_id=intent_id, - policy=policy, - ) if shadow and "record" in shadow: - legacy_status, legacy_recommendation = _legacy_status_and_recommendation(evidence) shadow_comparison = compare_shadow_to_legacy( - shadow=shadow["record"], - legacy_status=legacy_status, - legacy_recommendation=legacy_recommendation, - ) shadow_comparison["routing_mode"] = routing_config.mode @@ -653,168 +414,88 @@ def _evaluate_obligation( shadow_comparison["legacy_authoritative"] = True elif shadow and "error" in shadow: - shadow_comparison = { - "kind": "shadow_comparison", - "agreement": False, - "error": shadow["error"], - "legacy_authoritative": True, - "routing_mode": routing_config.mode, - } - - if use_cache and cache_dir is not None: - store_cached_evidence(cache_dir, key, evidence.model_dump(mode="json")) return _attach_execution_metadata( - evidence, - lane=lane, - data=data, - routing=precomputed_routing, - shadow_comparison=shadow_comparison, - intent_id=intent_id, - job_id=obligation.get("job_id"), - input_format=input_format, - routing_enforced=False, - ) - - - def execute_obligations( - obligations: list[dict[str, Any]], - routing_by_intent: Mapping[str, RoutingDecision | Mapping[str, Any] | None] | None = None, - *, - repo: str, - head_sha: str, - base_sha: str | None = None, - cache_dir: Path | None = None, - use_cache: bool = True, - parallel: bool = True, - policy: dict[str, Any] | None = None, - evidence_schema_version: str = "ovk.evidence.v3", - ) -> list[VerificationEvidence]: - """Evaluate obligations using one authoritative routing decision per intent.""" if not obligations: - return [] - - routing_plan = ensure_authoritative_routing( - obligations, - routing_by_intent, - policy=policy, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - ) - - if parallel and len(obligations) > 1: - with ThreadPoolExecutor(max_workers=min(len(obligations), 5)) as pool: - futures = [ - pool.submit( - _evaluate_obligation, - obligation, - routing_plan=routing_plan, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - cache_dir=cache_dir, - use_cache=use_cache, - policy=policy, - evidence_schema_version=evidence_schema_version, - ) - for obligation in obligations - ] return [future.result() for future in futures] - - return [ - _evaluate_obligation( - obligation, - routing_plan=routing_plan, - repo=repo, - head_sha=head_sha, - base_sha=base_sha, - cache_dir=cache_dir, - use_cache=use_cache, - policy=policy, - evidence_schema_version=evidence_schema_version, - ) - for obligation in obligations - ] - diff --git a/ovk/core/backend_aggregation.py b/ovk/core/backend_aggregation.py index fa3c894..df5ae16 100644 --- a/ovk/core/backend_aggregation.py +++ b/ovk/core/backend_aggregation.py @@ -84,10 +84,7 @@ def evaluate_fallback_acceptance( return True, False, attempt.termination if policy.fallback_backends and selection.backend not in policy.fallback_backends: return True, False, attempt.termination - if ( - policy.acceptable_fallback_guarantees - and result.guarantee_type not in policy.acceptable_fallback_guarantees - ): + if policy.acceptable_fallback_guarantees and result.guarantee_type not in policy.acceptable_fallback_guarantees: return True, False, attempt.termination if fallback_used: @@ -172,10 +169,7 @@ def aggregate_fail_dominant_v1( return AggregationOutcome( status=VerificationStatus.UNKNOWN, merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, - reason=( - "selected and executed backend sets differ; " - f"missing={missing}; unexpected={unexpected}" - ), + reason=(f"selected and executed backend sets differ; missing={missing}; unexpected={unexpected}"), quality_error=True, ) @@ -236,9 +230,7 @@ def aggregate_fail_dominant_v1( if any(item.status in non_pass for item in required_results): for item in optional_results: if item.status == VerificationStatus.PASS: - warnings.append( - f"optional backend {item.backend} passed but cannot upgrade required unknown/error" - ) + warnings.append(f"optional backend {item.backend} passed but cannot upgrade required unknown/error") return AggregationOutcome( status=VerificationStatus.UNKNOWN, merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, @@ -268,8 +260,7 @@ def aggregate_fail_dominant_v1( status=VerificationStatus.UNKNOWN, merge_recommendation=MergeRecommendation.REQUIRE_STRONGER_CHECK, reason=( - f"required result from {item.backend} uses guarantee " - f"{item.guarantee_type!r} outside acceptable set" + f"required result from {item.backend} uses guarantee {item.guarantee_type!r} outside acceptable set" ), fallback_used=fallback_used, fallback_accepted=resolved_fallback_accepted, diff --git a/ovk/core/backend_control_plane.py b/ovk/core/backend_control_plane.py index 12ca59b..036192e 100644 --- a/ovk/core/backend_control_plane.py +++ b/ovk/core/backend_control_plane.py @@ -19,7 +19,6 @@ from ovk.core.backend_aggregation import aggregate_results from ovk.core.backend_registry import BackendRegistry, BackendRegistryError -from ovk.core.bundle import content_digest from ovk.core.execution_budget import BackendWorker, LocalSubprocessWorker from ovk.core.execution_models import ( BackendEnvironmentFingerprint, @@ -318,14 +317,12 @@ def _execute_one( adapter = registry.require(selection_backend) if adapter.backend_id != selection_backend: raise BackendRegistryError( - f"adapter identity mismatch: expected {selection_backend}, " - f"got {adapter.backend_id}" + f"adapter identity mismatch: expected {selection_backend}, got {adapter.backend_id}" ) compiled = adapter.compile(obligation, routing) if compiled.backend != selection_backend: raise BackendRegistryError( - f"compiled backend {compiled.backend!r} does not match selection " - f"{selection_backend!r}" + f"compiled backend {compiled.backend!r} does not match selection {selection_backend!r}" ) if compiled.expected_guarantee != expected_guarantee and expected_guarantee: message = ( @@ -379,9 +376,7 @@ def _execute_one( if cached is not None: # Replay stored provenance; never re-infer native_execution. stored_attempt = cached.attempt - result = cached.normalized_result.model_copy( - update={"attempt_id": stored_attempt.attempt_id} - ) + result = cached.normalized_result.model_copy(update={"attempt_id": stored_attempt.attempt_id}) return stored_attempt, result, compiled raw = self._run_adapter(adapter, compiled, budget) @@ -414,9 +409,7 @@ def _execute_one( except Exception as exc: # noqa: BLE001 - isolate failures at backend boundary raw = _error_raw( backend=selection_backend, - backend_obligation_id=( - compiled.backend_obligation_id if compiled is not None else "uncompiled" - ), + backend_obligation_id=(compiled.backend_obligation_id if compiled is not None else "uncompiled"), stage="execute", exc=exc, started_at=started_at, @@ -440,7 +433,6 @@ def _execute_one( ) return attempt, result, compiled - def _run_adapter(self, adapter: Any, compiled: BackendObligation, budget: ExecutionBudget) -> RawBackendExecution: """Invoke adapter.run, threading the worker when the adapter accepts it.""" run = adapter.run diff --git a/ovk/core/backend_ids.py b/ovk/core/backend_ids.py index ba31555..3787e52 100644 --- a/ovk/core/backend_ids.py +++ b/ovk/core/backend_ids.py @@ -46,15 +46,9 @@ "lane-deployment", } ), - "self_protection": frozenset( - {"opa-native", "self-protection-deterministic", "lane-self-protection"} - ), - "authorization": frozenset( - {"z3-native", "authorization-deterministic", "lane-authorization"} - ), - "infrastructure": frozenset( - {"infrastructure-deterministic", "lane-infrastructure"} - ), + "self_protection": frozenset({"opa-native", "self-protection-deterministic", "lane-self-protection"}), + "authorization": frozenset({"z3-native", "authorization-deterministic", "lane-authorization"}), + "infrastructure": frozenset({"infrastructure-deterministic", "lane-infrastructure"}), "ci_secrets": frozenset({"ci-secrets-deterministic", "lane-ci-secrets"}), "deployment": frozenset({"deployment-deterministic", "lane-deployment"}), } diff --git a/ovk/core/backend_registry.py b/ovk/core/backend_registry.py index 5840247..adc31c1 100644 --- a/ovk/core/backend_registry.py +++ b/ovk/core/backend_registry.py @@ -34,8 +34,7 @@ def _validate_capability_manifest(manifest: BackendCapabilityManifest) -> None: report = validate_against_schema(payload, schema) if not report.valid: issues = "; ".join( - f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" - for issue in report.issues + f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" for issue in report.issues ) raise BackendRegistryError(f"capability manifest failed schema validation: {issues}") @@ -68,15 +67,11 @@ def register(self, adapter: BackendAdapter) -> None: raise BackendRegistryError(f"duplicate backend registration: {backend_id}") adapter_key = (adapter_id, adapter_version) if adapter_key in self._adapter_keys: - raise BackendRegistryError( - f"duplicate adapter identity: {adapter_id}@{adapter_version}" - ) + raise BackendRegistryError(f"duplicate adapter identity: {adapter_id}@{adapter_version}") manifest = adapter.manifest() if not isinstance(manifest, BackendCapabilityManifest): - raise BackendRegistryError( - f"adapter {backend_id!r} manifest() must return BackendCapabilityManifest" - ) + raise BackendRegistryError(f"adapter {backend_id!r} manifest() must return BackendCapabilityManifest") tool = manifest.tool if tool.adapter != adapter_id: raise BackendRegistryError( @@ -93,17 +88,13 @@ def register(self, adapter: BackendAdapter) -> None: if not manifest.supported_domains: raise BackendRegistryError(f"adapter {backend_id!r} must declare supported_domains") if not manifest.supported_property_kinds: - raise BackendRegistryError( - f"adapter {backend_id!r} must declare supported_property_kinds" - ) + raise BackendRegistryError(f"adapter {backend_id!r} must declare supported_property_kinds") _validate_capability_manifest(manifest) # Require compile/run surface present (Protocol methods). for method_name in ("can_handle", "compile", "fingerprint", "run", "normalize", "explain"): if not callable(getattr(adapter, method_name, None)): - raise BackendRegistryError( - f"adapter {backend_id!r} is missing required method {method_name}" - ) + raise BackendRegistryError(f"adapter {backend_id!r} is missing required method {method_name}") self._by_backend[backend_id] = adapter self._adapter_keys.add(adapter_key) @@ -142,8 +133,7 @@ def candidates( assessment = adapter.can_handle(obligation, context) if assessment.backend != adapter.backend_id: raise BackendRegistryError( - f"adapter {adapter.backend_id!r} returned assessment for " - f"backend {assessment.backend!r}" + f"adapter {adapter.backend_id!r} returned assessment for backend {assessment.backend!r}" ) assessments.append(assessment) return sorted( diff --git a/ovk/core/bundle.py b/ovk/core/bundle.py index ca101d9..489a83f 100644 --- a/ovk/core/bundle.py +++ b/ovk/core/bundle.py @@ -22,9 +22,7 @@ def _validate_bundle_inputs(evidence: list[VerificationEvidence]) -> None: subject = evidence[0].subject for index, item in enumerate(evidence[1:], start=1): if item.subject != subject: - raise ValueError( - f"evidence subject mismatch at index {index}: expected {subject}, got {item.subject}" - ) + raise ValueError(f"evidence subject mismatch at index {index}: expected {subject}, got {item.subject}") evidence_ids = [item.evidence_id for item in evidence] duplicate_ids = sorted({item for item in evidence_ids if evidence_ids.count(item) > 1}) if duplicate_ids: diff --git a/ovk/core/compiler_bridge.py b/ovk/core/compiler_bridge.py index bcd90ec..5e5cdb0 100644 --- a/ovk/core/compiler_bridge.py +++ b/ovk/core/compiler_bridge.py @@ -398,8 +398,7 @@ def register_cbmc_project(data: dict[str, Any]) -> tuple[CbmcProject, Abstractio confidence=0.4, extracted_elements=extracted, expected_elements=None, - warnings=list(project.warnings) - + ["CBMC materials registered without project-grounded strict eligibility"], + warnings=list(project.warnings) + ["CBMC materials registered without project-grounded strict eligibility"], ) compiler_id = "ovk.cbmc.harness_or_cdb.v1" else: diff --git a/ovk/core/context.py b/ovk/core/context.py index 4bb9179..fee1fa2 100644 --- a/ovk/core/context.py +++ b/ovk/core/context.py @@ -66,10 +66,7 @@ def _validate_policy_mapping(loaded: object, *, source: str) -> dict[str, Any]: for error in errors: location = "/".join(str(part) for part in error.path) or "$" formatted.append(f"{location}: {error.message}") - raise ValueError( - f"OVK verification policy from {source} failed schema validation: " - + "; ".join(formatted) - ) + raise ValueError(f"OVK verification policy from {source} failed schema validation: " + "; ".join(formatted)) return dict(loaded) @@ -170,9 +167,7 @@ def budget_from_policy(policy: dict[str, Any]) -> VerificationBudget: denied = normalize_denied_backends(denied_raw) allowed_set = frozenset(allowed) if allowed is not None else None denied_set = frozenset(denied) - max_wall = float( - budget_section.get("max_wall_time_seconds", policy.get("max_wall_time_seconds", 30.0)) - ) + max_wall = float(budget_section.get("max_wall_time_seconds", policy.get("max_wall_time_seconds", 30.0))) max_memory = int(budget_section.get("max_memory_mb", policy.get("max_memory_mb", 512))) routing_section = policy.get("routing", {}) prefer_deterministic = False diff --git a/ovk/core/counterexample_translator.py b/ovk/core/counterexample_translator.py index 739d3da..1abefab 100644 --- a/ovk/core/counterexample_translator.py +++ b/ovk/core/counterexample_translator.py @@ -211,17 +211,14 @@ def _under(root: Path) -> bool: return False if not (_under(cwd) or _under(verification_root) or _under(temp_root)): - raise ValueError( - f"refusing to write generated tests outside workspace or temp: {resolved}" - ) + raise ValueError(f"refusing to write generated tests outside workspace or temp: {resolved}") resolved.mkdir(parents=True, exist_ok=True) written: list[Path] = [] for index, artifact in enumerate(generate_regression_artifacts(bundle)): - failure_mode = "".join( - ch if ch.isalnum() or ch in {"_", "-"} else "_" - for ch in str(artifact["failure_mode"]) - )[:80] + failure_mode = "".join(ch if ch.isalnum() or ch in {"_", "-"} else "_" for ch in str(artifact["failure_mode"]))[ + :80 + ] json_path = resolved / f"regression_{index}_{failure_mode}.json" # Ensure no path traversal via failure_mode. if json_path.resolve().parent != resolved: diff --git a/ovk/core/decision.py b/ovk/core/decision.py index 5633c5e..ba8512d 100644 --- a/ovk/core/decision.py +++ b/ovk/core/decision.py @@ -7,11 +7,7 @@ def evidence_has_status(bundle: EvidenceBundle, status: VerificationStatus) -> bool: """Return true if any backend claim in the bundle has the given status.""" - return any( - claim.status == status - for evidence in bundle.evidence - for claim in evidence.backend_claims - ) + return any(claim.status == status for evidence in bundle.evidence for claim in evidence.backend_claims) def evidence_has_unknown_like(bundle: EvidenceBundle) -> bool: @@ -21,11 +17,7 @@ def evidence_has_unknown_like(bundle: EvidenceBundle) -> bool: VerificationStatus.ERROR, VerificationStatus.SKIPPED, } - return any( - claim.status in unknown_like - for evidence in bundle.evidence - for claim in evidence.backend_claims - ) + return any(claim.status in unknown_like for evidence in bundle.evidence for claim in evidence.backend_claims) def _unknown_like_recommendation( @@ -90,11 +82,16 @@ def decide_with_reason( ) -> dict[str, str]: """Return merge recommendation and human-readable reason for bundle construction.""" recommendation = decide(bundle, enforce=enforce, default_on_unknown=default_on_unknown) - from_unknown = recommendation in { - MergeRecommendation.BLOCK, - MergeRecommendation.REQUIRE_HUMAN_REVIEW, - MergeRecommendation.ALLOW_WITH_WARNING, - } and evidence_has_unknown_like(bundle) and not evidence_has_status(bundle, VerificationStatus.FAIL) + from_unknown = ( + recommendation + in { + MergeRecommendation.BLOCK, + MergeRecommendation.REQUIRE_HUMAN_REVIEW, + MergeRecommendation.ALLOW_WITH_WARNING, + } + and evidence_has_unknown_like(bundle) + and not evidence_has_status(bundle, VerificationStatus.FAIL) + ) return { "merge_recommendation": recommendation.value, "reason": _decision_reason(recommendation, from_unknown=from_unknown), diff --git a/ovk/core/diff_iac.py b/ovk/core/diff_iac.py index c9a8447..7e26d74 100644 --- a/ovk/core/diff_iac.py +++ b/ovk/core/diff_iac.py @@ -17,7 +17,9 @@ def _is_iac_path(path: str) -> bool: suffix = PurePosixPath(normalized).suffix if suffix in IAC_SUFFIXES: return True - return any(marker in normalized for marker in ("/k8s/", "/kubernetes/", "/deploy/", "deployment")) or normalized.startswith(("k8s/", "kubernetes/")) + return any( + marker in normalized for marker in ("/k8s/", "/kubernetes/", "/deploy/", "deployment") + ) or normalized.startswith(("k8s/", "kubernetes/")) def _iam_policy_is_overly_permissive(resource_type: str, lines: list[str]) -> bool: @@ -27,9 +29,9 @@ def _iam_policy_is_overly_permissive(resource_type: str, lines: list[str]) -> bo joined = "\n".join(lines).lower() if "iam_policy" not in resource_type.lower(): return False - if 'principal' in joined and "*" in joined and ("admin" in joined or "action" in joined): + if "principal" in joined and "*" in joined and ("admin" in joined or "action" in joined): return True - return 'action' in joined and 'resource' in joined and "*" in joined + return "action" in joined and "resource" in joined and "*" in joined def _parse_tf_hunk(content: str) -> dict | None: diff --git a/ovk/core/doctor.py b/ovk/core/doctor.py index 02ee907..962f9c5 100644 --- a/ovk/core/doctor.py +++ b/ovk/core/doctor.py @@ -122,7 +122,9 @@ def _check_manifest_example() -> DoctorCheck: try: data = read_json_file(manifest) lanes = data.get("lanes", []) - return DoctorCheck("example_manifest", isinstance(lanes, list) and len(lanes) >= 5, f"{len(lanes)} lanes in example manifest") + return DoctorCheck( + "example_manifest", isinstance(lanes, list) and len(lanes) >= 5, f"{len(lanes)} lanes in example manifest" + ) except Exception as error: # noqa: BLE001 return DoctorCheck("example_manifest", False, str(error)) diff --git a/ovk/core/evidence_from_execution.py b/ovk/core/evidence_from_execution.py index 414f108..3dc5fe0 100644 --- a/ovk/core/evidence_from_execution.py +++ b/ovk/core/evidence_from_execution.py @@ -1,15 +1,11 @@ """Convert control-plane execution records into VerificationEvidence.""" - - from __future__ import annotations - from typing import Any - from ovk.compilers.authorization import CoveragePolicy, strict_allow_permitted from ovk.core.bundle import content_digest @@ -21,29 +17,16 @@ from ovk.core.models import BackendClaim, MergeRecommendation, VerificationEvidence, VerificationStatus - - - def execution_record_to_evidence( - record: ObligationExecutionRecord, - *, - author_type: str = "unknown", - agent: str = "unknown", - task: str = "unknown", - routing_enforced: bool = False, - schema_version: str = "ovk.evidence.v2", - coverage_policy: CoveragePolicy | None = None, - ) -> VerificationEvidence: - """Project an obligation execution record into public evidence.""" obligation = record.obligation @@ -53,165 +36,88 @@ def execution_record_to_evidence( counterexamples: list[dict[str, Any]] = [] for result in record.results: - counterexamples.extend(result.counterexamples) - - claims = [ - BackendClaim( - backend=result.backend, - guarantee_type=result.guarantee_type, - status=result.status, - assumptions=list(result.assumptions), - limits=list(result.limits), - adapter_version=next( - - ( - - item.adapter_version - - for item in record.backend_obligations - - if item.backend == result.backend - - ), - + (item.adapter_version for item in record.backend_obligations if item.backend == result.backend), None, - ), - ) - for result in sorted(record.results, key=lambda item: item.backend) - ] if not claims: - claims = [ - BackendClaim( - backend="none", - guarantee_type="none", - status=VerificationStatus.UNKNOWN, - assumptions=["No backend produced a claim."], - limits=["Absence of backend evidence cannot allow."], - ) - ] - - material_payloads = [item.model_dump(mode="json") for item in obligation.materials] material_set_digest = material_set_digest_for_obligation(obligation) - - artifacts: list[dict[str, Any]] = [] for result in record.results: - artifacts.extend(result.generated_artifacts) for item in record.open_obligations: - artifacts.append(dict(item)) artifacts.append( - { - "kind": "routing_enforced", - "value": routing_enforced, - "routing_id": routing.routing_id, - "obligation_id": obligation.obligation_id, - } - ) artifacts.append( - { - "kind": "control_plane_trace", - "compiler": { - "compiler_id": obligation.compiler_id, - "compiler_version": obligation.compiler_version, - }, - "coverage": obligation.coverage.model_dump(mode="json"), - "material_set_digest": material_set_digest, - "routing_id": routing.routing_id, - "requested_backends": list(routing.requested), - "eligible_backends": [item.backend for item in routing.eligible], - "selected_backends": [item.backend for item in routing.selected], - "attempted_backends": [item.backend for item in record.attempts], - "executed_backends": [item.backend for item in record.results], - "execution_attempts": [item.model_dump(mode="json") for item in record.attempts], - "routing_enforced": routing_enforced, - "aggregation_policy": routing.aggregation_policy, - } - ) if obligation.compiler_id: - artifacts.append( - { - "kind": "compiler_identity", - "compiler_id": obligation.compiler_id, - "compiler_version": obligation.compiler_version, - "coverage": obligation.coverage.model_dump(mode="json"), - "materials": material_payloads, - "material_set_digest": material_set_digest, - } - ) - - recommendation = record.merge_recommendation aggregation_reason = record.aggregation_reason @@ -221,134 +127,69 @@ def execution_record_to_evidence( allow_ok = obligation.abstraction.get("strict_allow_permitted") if allow_ok is None: - allow_ok = strict_allow_permitted(obligation.coverage, policy) if recommendation == MergeRecommendation.ALLOW and not allow_ok: - recommendation = MergeRecommendation.REQUIRE_HUMAN_REVIEW - aggregation_reason = ( - - f"{aggregation_reason}; incomplete abstraction cannot allow under strict coverage" - - ) + aggregation_reason = f"{aggregation_reason}; incomplete abstraction cannot allow under strict coverage" artifacts.append( - { - "kind": "incomplete_abstraction", - "coverage": obligation.coverage.model_dump(mode="json"), - "reason": "strict allow blocked unless coverage complete or policy accepts partial", - } - ) - - decision = { - "merge_recommendation": recommendation.value, - "human_review_required": recommendation.value != "allow", - "aggregation_reason": aggregation_reason, - "routing_enforced": routing_enforced, - "fallback_used": record.fallback_used, - "fallback_accepted": record.fallback_accepted, - "fallback_cause": record.fallback_cause, - } - - evidence_id = content_digest( - { - "obligation_id": obligation.obligation_id, - "routing_id": routing.routing_id, - "material_set_digest": material_set_digest, - "results": [claim.model_dump(mode="json") for claim in claims], - } - )[:24] - - return VerificationEvidence( - evidence_id=f"ev-{evidence_id}", - schema_version=schema_version, - subject={key: value for key, value in obligation.subject.model_dump(mode="json").items() if value is not None}, - change_origin={"author_type": author_type, "agent": agent, "task": task}, - intent={ - "intent_id": obligation.intent_id, - "title": obligation.intent_id, - "risk": {"severity": obligation.severity.value}, - }, - backend_claims=claims, - counterexamples=counterexamples, - generated_artifacts=artifacts, - decision=decision, - obligation_id=obligation.obligation_id, - routing_id=routing.routing_id, - material_set_digest=material_set_digest if schema_version.endswith(".v3") else None, - compiler={ - "compiler_id": obligation.compiler_id, - "compiler_version": obligation.compiler_version, - }, - materials=material_payloads, - coverage=obligation.coverage.model_dump(mode="json"), - requested_backends=list(routing.requested), - eligible_backends=[item.backend for item in routing.eligible], - selected_backends=[item.backend for item in routing.selected], - attempted_backends=[item.backend for item in record.attempts], - executed_backends=[item.backend for item in record.results], - execution_attempts=[item.model_dump(mode="json") for item in record.attempts], - aggregation_policy=routing.aggregation_policy, - routing_enforced=routing_enforced, - ) - diff --git a/ovk/core/evidence_invariants.py b/ovk/core/evidence_invariants.py index 50dea1c..7eda9bf 100644 --- a/ovk/core/evidence_invariants.py +++ b/ovk/core/evidence_invariants.py @@ -312,7 +312,11 @@ def _check_control_plane_invariants(bundle: EvidenceBundle) -> list[EvidenceInva ) ) - if evidence.aggregation_policy and evidence.decision.get("aggregation_reason") is None and evidence.routing_enforced: + if ( + evidence.aggregation_policy + and evidence.decision.get("aggregation_reason") is None + and evidence.routing_enforced + ): # aggregation_reason is optional on legacy decision dicts; required for enforced v2. if "aggregation_reason" not in evidence.decision: issues.append( diff --git a/ovk/core/infrastructure_compiler.py b/ovk/core/infrastructure_compiler.py index a78c1b4..19e801d 100644 --- a/ovk/core/infrastructure_compiler.py +++ b/ovk/core/infrastructure_compiler.py @@ -75,8 +75,7 @@ def compile_infrastructure_obligation( coverage=coverage, acceptable_guarantees=["exposure_graph_check"], required_capabilities=["infrastructure"], - policy_digest=policy_digest - or content_digest({"lane": "infrastructure", "policy": policy or {}}), + policy_digest=policy_digest or content_digest({"lane": "infrastructure", "policy": policy or {}}), ) return provisional.model_copy(update={"obligation_id": compute_obligation_id(provisional)}) diff --git a/ovk/core/lane_compiler.py b/ovk/core/lane_compiler.py index c8e0e9b..c1a4214 100644 --- a/ovk/core/lane_compiler.py +++ b/ovk/core/lane_compiler.py @@ -60,10 +60,14 @@ def compile_lane_inputs_from_plan( if diff_text and is_unified_diff(diff_text): if "ci_secrets" in lanes_needed: for index, data in enumerate(workflow_inputs_from_diff(diff_text)): - jobs.append({"lane": "ci_secrets", "data": data, "input_format": "infra", "job_id": f"ci_secrets_{index}"}) + jobs.append( + {"lane": "ci_secrets", "data": data, "input_format": "infra", "job_id": f"ci_secrets_{index}"} + ) if "authorization" in lanes_needed: for index, data in enumerate(authorization_inputs_from_diff(diff_text)): - jobs.append({"lane": "authorization", "data": data, "input_format": "infra", "job_id": f"authorization_{index}"}) + jobs.append( + {"lane": "authorization", "data": data, "input_format": "infra", "job_id": f"authorization_{index}"} + ) if "infrastructure" in lanes_needed: for index, item in enumerate(infra_inputs_from_diff(diff_text)): jobs.append( @@ -76,7 +80,9 @@ def compile_lane_inputs_from_plan( ) if "deployment" in lanes_needed: for index, data in enumerate(deployment_inputs_from_diff(diff_text)): - jobs.append({"lane": "deployment", "data": data, "input_format": "infra", "job_id": f"deployment_{index}"}) + jobs.append( + {"lane": "deployment", "data": data, "input_format": "infra", "job_id": f"deployment_{index}"} + ) if "backend" in lanes_needed: candidate_intents = set(plan.get("candidate_intents", [])) for index, data in enumerate(cbmc_inputs_from_diff(diff_text)): @@ -97,7 +103,9 @@ def compile_lane_inputs_from_plan( if isinstance(suggested.get("ci_secrets"), list): existing = sum(1 for job in jobs if job["lane"] == "ci_secrets") for index, data in enumerate(suggested["ci_secrets"][existing:]): - jobs.append({"lane": "ci_secrets", "data": data, "input_format": "infra", "job_id": f"ci_secrets_suggested_{index}"}) + jobs.append( + {"lane": "ci_secrets", "data": data, "input_format": "infra", "job_id": f"ci_secrets_suggested_{index}"} + ) return jobs diff --git a/ovk/core/multi_lane.py b/ovk/core/multi_lane.py index f058fed..e0331f8 100644 --- a/ovk/core/multi_lane.py +++ b/ovk/core/multi_lane.py @@ -39,8 +39,7 @@ def _validate_lane_input(data: dict[str, Any], schema_name: str, *, lane: str) - report = validate_against_schema(data, schema) if not report.valid: issues = "; ".join( - f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" - for issue in report.issues + f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" for issue in report.issues ) raise ValueError(f"{lane} input failed schema validation: {issues}") @@ -138,8 +137,7 @@ def load_verification_manifest(path: Path, *, validate: bool = True) -> dict[str report = validate_against_schema(manifest, schema) if not report.valid: issues = "; ".join( - f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" - for issue in report.issues + f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" for issue in report.issues ) raise ValueError(f"verification manifest failed schema validation: {issues}") return manifest @@ -200,9 +198,7 @@ def _evaluate_manifest_entry( return None input_path = _resolve_manifest_file(manifest_root, str(input_value), field="input") policy_path = ( - _resolve_manifest_file(manifest_root, str(entry["policy"]), field="policy") - if entry.get("policy") - else None + _resolve_manifest_file(manifest_root, str(entry["policy"]), field="policy") if entry.get("policy") else None ) data = read_json_file(input_path) evidence = evaluate_lane( @@ -220,9 +216,7 @@ def _evaluate_manifest_entry( "input_format": str(entry.get("input_format", "infra")), "policy": entry.get("policy"), } - return evidence.model_copy( - update={"evidence_id": f"{evidence.evidence_id}-{content_digest(identity)[:12]}"} - ) + return evidence.model_copy(update={"evidence_id": f"{evidence.evidence_id}-{content_digest(identity)[:12]}"}) def run_verification_manifest( diff --git a/ovk/core/native_backend_probe.py b/ovk/core/native_backend_probe.py index 6eeb06a..032fe6d 100644 --- a/ovk/core/native_backend_probe.py +++ b/ovk/core/native_backend_probe.py @@ -218,7 +218,9 @@ def probe_all_native_backends() -> list[NativeBackendSummary]: continue binary_present = any(item.binary_present for item in results) fixture_matches = all(item.runtime_status == item.oracle_status for item in results) - native_used = all(item.used_native_binary for item in results if item.binary_present) if binary_present else False + native_used = ( + all(item.used_native_binary for item in results if item.binary_present) if binary_present else False + ) if backend in TIER1_NATIVE_EXECUTION_BACKENDS and binary_present: native_used = all(item.used_native_binary for item in results) summaries.append( diff --git a/ovk/core/obligation_compiler.py b/ovk/core/obligation_compiler.py index 005d199..c054bc2 100644 --- a/ovk/core/obligation_compiler.py +++ b/ovk/core/obligation_compiler.py @@ -48,10 +48,13 @@ def compile( obligations: list[dict[str, Any]] = [] for job in jobs: lane = str(job["lane"]) - intent_id = str(job.get("intent_id") or next( - (intent for intent, mapped in self._intent_to_lane.items() if mapped == lane), - lane, - )) + intent_id = str( + job.get("intent_id") + or next( + (intent for intent, mapped in self._intent_to_lane.items() if mapped == lane), + lane, + ) + ) obligations.append( { "intent_id": intent_id, diff --git a/ovk/core/output_validation.py b/ovk/core/output_validation.py index d258a64..3f9a561 100644 --- a/ovk/core/output_validation.py +++ b/ovk/core/output_validation.py @@ -69,8 +69,7 @@ def missing_release_layout_schema_coverage(layout: dict[str, Any]) -> list[str]: def _issues_from_pydantic(error: ValidationError) -> list[ValidationIssue]: return [ - ValidationIssue(path=[str(part) for part in issue["loc"]], message=issue["msg"]) - for issue in error.errors() + ValidationIssue(path=[str(part) for part in issue["loc"]], message=issue["msg"]) for issue in error.errors() ] @@ -113,10 +112,7 @@ def validate_generated_file(path: Path, kind: str) -> ValidationReport: def _format_file_validation_failures(path: Path, report: ValidationReport) -> list[str]: - return [ - f"{path.name} validation at {issue.path}: {issue.message}" - for issue in report.issues - ] + return [f"{path.name} validation at {issue.path}: {issue.message}" for issue in report.issues] def validate_output_directory(root: Path) -> list[str]: diff --git a/ovk/core/policy_config.py b/ovk/core/policy_config.py index 4ffa5b2..33d7f34 100644 --- a/ovk/core/policy_config.py +++ b/ovk/core/policy_config.py @@ -6,9 +6,7 @@ from ovk.core.router import RoutingConfig, routing_config_from_policy -VALID_DEFAULT_ON_UNKNOWN = frozenset( - {"require_human_review", "block", "allow_with_warning"} -) +VALID_DEFAULT_ON_UNKNOWN = frozenset({"require_human_review", "block", "allow_with_warning"}) def resolve_default_on_unknown(policy: dict[str, Any] | None) -> str: diff --git a/ovk/core/provenance.py b/ovk/core/provenance.py index 2275261..20deb1d 100644 --- a/ovk/core/provenance.py +++ b/ovk/core/provenance.py @@ -87,17 +87,13 @@ def build_provenance_statement( "control_plane": { "obligation_ids": [item.obligation_id for item in bundle.evidence if item.obligation_id], "routing_ids": [item.routing_id for item in bundle.evidence if item.routing_id], - "material_set_digests": [ - item.material_set_digest for item in bundle.evidence if item.material_set_digest - ], + "material_set_digests": [item.material_set_digest for item in bundle.evidence if item.material_set_digest], "routing_enforced": [bool(item.routing_enforced) for item in bundle.evidence], "compilers": [item.compiler for item in bundle.evidence if item.compiler], "coverage": [item.coverage for item in bundle.evidence if item.coverage], "selected_backends": [item.selected_backends for item in bundle.evidence], "executed_backends": [item.executed_backends for item in bundle.evidence], - "guarantees": [ - [claim.guarantee_type for claim in item.backend_claims] for item in bundle.evidence - ], + "guarantees": [[claim.guarantee_type for claim in item.backend_claims] for item in bundle.evidence], }, "materials": [material_entry(path, workspace=workspace) for path in materials or []], "invocation": invocation diff --git a/ovk/core/release_preflight_report.py b/ovk/core/release_preflight_report.py index 84b079b..2546e1d 100644 --- a/ovk/core/release_preflight_report.py +++ b/ovk/core/release_preflight_report.py @@ -121,9 +121,7 @@ def _check_smoke_quality_reports() -> list[str]: make_bundle( [ evaluate_ci_secrets_exposure( - read_json_file( - resource_path("examples", "ci_secrets", "input_secrets_safe.json") - ), + read_json_file(resource_path("examples", "ci_secrets", "input_secrets_safe.json")), repo="smoke/repo", head_sha="smoke-head", ) @@ -189,13 +187,9 @@ def _check_pilot_program() -> list[str]: failures: list[str] = [] for result in report["results"]: if not result["passed"]: - failures.append( - f"pilot manifest {result['name']} did not allow: {result['merge_recommendation']}" - ) + failures.append(f"pilot manifest {result['name']} did not allow: {result['merge_recommendation']}") if report["manifests_passed"] != report["manifests_total"]: - failures.append( - f"pilot program passed {report['manifests_passed']}/{report['manifests_total']} manifests" - ) + failures.append(f"pilot program passed {report['manifests_passed']}/{report['manifests_total']} manifests") return failures @@ -271,9 +265,7 @@ def build_release_preflight_report() -> PreflightReport: check_from_failures("release_layout_schema_coverage", _check_release_layout_schema_coverage()), check_from_failures("adapter_capabilities", _check_adapter_capabilities()), ), - optional_checks=( - check_from_failures("pilot_metrics_dry_run", _check_pilot_metrics_dry_run()), - ), + optional_checks=(check_from_failures("pilot_metrics_dry_run", _check_pilot_metrics_dry_run()),), ) diff --git a/ovk/core/render.py b/ovk/core/render.py index 521c451..a754a61 100644 --- a/ovk/core/render.py +++ b/ovk/core/render.py @@ -12,7 +12,8 @@ def _section_kind(evidence: VerificationEvidence) -> str: artifacts = evidence.generated_artifacts or [] kinds = {str(item.get("kind")) for item in artifacts if isinstance(item, dict)} if "incomplete_abstraction" in kinds or ( - isinstance(evidence.coverage, dict) and evidence.coverage.get("status") in {"partial", "unknown"} + isinstance(evidence.coverage, dict) + and evidence.coverage.get("status") in {"partial", "unknown"} and recommendation == "require_human_review" and "incomplete" in str(evidence.decision.get("aggregation_reason", "")) ): @@ -127,15 +128,14 @@ def render_evidence_markdown(evidence: VerificationEvidence) -> str: if hint.get("line_hunk") is not None: location += f", line {hint['line_hunk']}" location += ")" - lines.append( - f"- `{hint['fix_class']}`: {hint['suggested_action']}{location}" - ) + lines.append(f"- `{hint['fix_class']}`: {hint['suggested_action']}{location}") open_items = [ item for item in (evidence.generated_artifacts or []) if isinstance(item, dict) - and item.get("kind") in {"backend_disagreement", "quality_error", "incomplete_abstraction", "aggregation_warning"} + and item.get("kind") + in {"backend_disagreement", "quality_error", "incomplete_abstraction", "aggregation_warning"} ] if open_items: lines.extend(["", "Open obligations:"]) diff --git a/ovk/core/result_cache.py b/ovk/core/result_cache.py index 85f5b3d..574887f 100644 --- a/ovk/core/result_cache.py +++ b/ovk/core/result_cache.py @@ -25,7 +25,6 @@ BackendEnvironmentFingerprint, BackendObligation, CachedBackendExecution, - ExecutionAttempt, NormalizedBackendResult, RoutingDecision, VerificationObligation, diff --git a/ovk/core/router.py b/ovk/core/router.py index 7ffa183..067ec4f 100644 --- a/ovk/core/router.py +++ b/ovk/core/router.py @@ -110,7 +110,9 @@ def routing_config_from_policy(policy: Mapping[str, Any] | None) -> RoutingConfi else "primary_with_optional_corroboration" ) enforced_raw = section.get("enforced_lanes") or policy.get("enforced_lanes") or [] - enforced = frozenset(str(item) for item in enforced_raw) if isinstance(enforced_raw, (list, tuple, set)) else frozenset() + enforced = ( + frozenset(str(item) for item in enforced_raw) if isinstance(enforced_raw, (list, tuple, set)) else frozenset() + ) return RoutingConfig( mode=mode, strategy=strategy, @@ -318,8 +320,7 @@ def select_primary_with_optional_corroboration( primary_candidates = [ item for item in eligible_sorted - if item.support == "partial" - and not any("incomplete coverage" in reason for reason in item.reasons) + if item.support == "partial" and not any("incomplete coverage" in reason for reason in item.reasons) ] if primary_candidates: @@ -478,13 +479,7 @@ def _manifest_assessment( support: Literal["supported", "partial", "unsupported", "unavailable"] = "supported" # Explicit relevance from domain/kind match replaces the former constant 1.0 term. relevance = 1.0 if kind_match else 0.5 - score = ( - relevance - + guarantee_strength - + (0.15 * historical_success) - + surface_bonus - - cost - ) + score = relevance + guarantee_strength + (0.15 * historical_success) + surface_bonus - cost reasons.append(f"supports domain {domain} and property kind {property_kind}") elif domain_match: support = "partial" diff --git a/ovk/core/routing_pipeline.py b/ovk/core/routing_pipeline.py index 7818579..12d0376 100644 --- a/ovk/core/routing_pipeline.py +++ b/ovk/core/routing_pipeline.py @@ -40,6 +40,7 @@ def intent_id_for_obligation(obligation: dict[str, Any]) -> str: lane = str(obligation["lane"]) return str(obligation.get("intent_id") or LANE_TO_INTENT.get(lane, lane)) + RegistryBuilder = Callable[[], BackendRegistry] CompilerFn = Callable[..., VerificationObligation] @@ -335,7 +336,5 @@ def require_routing_decision( if decision is not None: return decision if routing_enforced_for_lane(policy, lane): - raise RuntimeError( - f"enforced lane {lane!r} requires authoritative RoutingDecision for intent {intent_id!r}" - ) + raise RuntimeError(f"enforced lane {lane!r} requires authoritative RoutingDecision for intent {intent_id!r}") raise RuntimeError(f"missing routing decision for intent {intent_id!r}") diff --git a/ovk/core/schema_validation.py b/ovk/core/schema_validation.py index 8ce7e80..00f87f2 100644 --- a/ovk/core/schema_validation.py +++ b/ovk/core/schema_validation.py @@ -72,7 +72,6 @@ def require_schema_valid( if report.valid: return issues = "; ".join( - f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" - for issue in report.issues + f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" for issue in report.issues ) raise ValueError(f"{context} failed schema validation: {issues}") diff --git a/ovk/core/sigstore_signing.py b/ovk/core/sigstore_signing.py index 63465ab..cb9f48b 100644 --- a/ovk/core/sigstore_signing.py +++ b/ovk/core/sigstore_signing.py @@ -39,9 +39,7 @@ def _configured_trust_identity() -> tuple[str, str]: identity = os.environ.get(COSIGN_IDENTITY_ENV, "").strip() issuer = os.environ.get(COSIGN_ISSUER_ENV, "").strip() if not identity or not issuer: - raise RuntimeError( - "OVK Sigstore signing requires OVK_COSIGN_IDENTITY and OVK_COSIGN_ISSUER" - ) + raise RuntimeError("OVK Sigstore signing requires OVK_COSIGN_IDENTITY and OVK_COSIGN_ISSUER") return identity, issuer diff --git a/ovk/core/source_profiles.py b/ovk/core/source_profiles.py index cb3d2d3..3486c84 100644 --- a/ovk/core/source_profiles.py +++ b/ovk/core/source_profiles.py @@ -68,12 +68,7 @@ def source_profile_strict_eligible( enforcement_test_present: bool, ) -> bool: """Return True when a template may claim source_profile_strict_eligible.""" - return ( - is_known_source_profile(profile_id) - and materials_trusted - and coverage_complete - and enforcement_test_present - ) + return is_known_source_profile(profile_id) and materials_trusted and coverage_complete and enforcement_test_present def profiles_from_policy(policy: dict[str, Any] | None, *, lane: str) -> list[str]: diff --git a/ovk/core/template_conformance.py b/ovk/core/template_conformance.py index bd3157e..17e17f2 100644 --- a/ovk/core/template_conformance.py +++ b/ovk/core/template_conformance.py @@ -215,7 +215,10 @@ def semantic_status_v2(self) -> ProductionStatusV2: enforcement_test_present=evidence.enforcement_test_present, ): return "source_profile_strict_eligible" - if self.production_status in {"strict_eligible", "advisory", "experimental"} and not self.missing_executable_links: + if ( + self.production_status in {"strict_eligible", "advisory", "experimental"} + and not self.missing_executable_links + ): return "executable_advisory" if self.production_status in {"advisory", "experimental"}: return "executable_advisory" @@ -312,9 +315,7 @@ def classify_template( # Honest downgrade for native-named templates without executable catalog entry. if claimed and intent_id not in EXECUTABLE_CATALOG: status = _min_status(status, "catalog_only") - notes.append( - "downgraded unsupported public executable claim for backends: " + ", ".join(claimed) - ) + notes.append("downgraded unsupported public executable claim for backends: " + ", ".join(claimed)) if template.get("deprecated") is True: status = "deprecated" @@ -328,13 +329,7 @@ def classify_template( relative = intent_path.relative_to(repo_root).as_posix() if intent_path.is_absolute() else intent_path.as_posix() evidence = template.get("acceptable_evidence") or [] - evidence_kinds = sorted( - { - str(item.get("kind")) - for item in evidence - if isinstance(item, dict) and item.get("kind") - } - ) + evidence_kinds = sorted({str(item.get("kind")) for item in evidence if isinstance(item, dict) and item.get("kind")}) risk = template.get("risk") if isinstance(template.get("risk"), dict) else {} prop = template.get("property") if isinstance(template.get("property"), dict) else {} externally_calibrated = bool(template.get("externally_calibrated") is True) @@ -391,9 +386,7 @@ def build_conformance_matrix(repo_root: Path, templates_dir: Path | None = None) "counts_by_status": dict(sorted(by_status.items())), "counts_by_status_v2": dict(sorted(by_status_v2.items())), "counts_by_domain": dict(sorted(by_domain.items())), - "source_profile_evidence": { - intent: item.as_dict() for intent, item in sorted(profile_evidence.items()) - }, + "source_profile_evidence": {intent: item.as_dict() for intent, item in sorted(profile_evidence.items())}, "templates": [row.to_dict() for row in rows], } return payload @@ -436,9 +429,7 @@ def validate_matrix(matrix: dict[str, Any]) -> list[str]: ) missing = row.get("missing_executable_links") or [] if status in {"strict_eligible", "advisory"} and missing and status == "strict_eligible": - failures.append( - f"{row.get('intent_id')}: strict_eligible requires empty missing_executable_links" - ) + failures.append(f"{row.get('intent_id')}: strict_eligible requires empty missing_executable_links") # catalog_only is mandatory when required links are incomplete and no # experimental/advisory catalog path was registered. if ( diff --git a/ovk/mcp_server.py b/ovk/mcp_server.py index 43bebe5..e1ed7a7 100644 --- a/ovk/mcp_server.py +++ b/ovk/mcp_server.py @@ -126,11 +126,7 @@ def explain_result(evidence_bundle: dict[str, Any]) -> dict[str, Any]: from ovk.core.counterexample_translator import repair_hint_for_counterexample bundle = EvidenceBundle.model_validate(evidence_bundle) - counterexamples = [ - counterexample - for evidence in bundle.evidence - for counterexample in evidence.counterexamples - ] + counterexamples = [counterexample for evidence in bundle.evidence for counterexample in evidence.counterexamples] repair_hints = [repair_hint_for_counterexample(item) for item in counterexamples] return { "bundle_id": bundle.bundle_id, diff --git a/scripts/expand_ovk_library.py b/scripts/expand_ovk_library.py index 1510643..1590c9c 100644 --- a/scripts/expand_ovk_library.py +++ b/scripts/expand_ovk_library.py @@ -17,7 +17,12 @@ DOMAIN_CONFIG: dict[str, dict[str, Any]] = { "authorization": { "kinds": ["access_control", "safety", "invariant"], - "failure_modes": ["middleware_skipped", "route_group_unprotected", "policy_default_allow", "role_check_removed"], + "failure_modes": [ + "middleware_skipped", + "route_group_unprotected", + "policy_default_allow", + "role_check_removed", + ], "evidence": [ {"kind": "policy_check", "minimum_confidence": "medium"}, {"kind": "smt_counterexample", "minimum_confidence": "medium"}, @@ -93,7 +98,11 @@ {"kind": "model_check", "minimum_confidence": "high"}, {"kind": "smt_counterexample", "minimum_confidence": "medium"}, ], - "risk": {"severity": "critical", "likelihood": "medium", "rationale": "Memory safety regressions in auth paths are exploitable."}, + "risk": { + "severity": "critical", + "likelihood": "medium", + "rationale": "Memory safety regressions in auth paths are exploitable.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -113,7 +122,11 @@ }, "failure_modes": ["policy_default_allow", "wildcard_principal", "privilege_escalation"], "acceptable_evidence": [{"kind": "policy_check", "minimum_confidence": "high"}], - "risk": {"severity": "critical", "likelihood": "medium", "rationale": "IAM regressions expose cloud control planes."}, + "risk": { + "severity": "critical", + "likelihood": "medium", + "rationale": "IAM regressions expose cloud control planes.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -133,7 +146,11 @@ }, "failure_modes": ["public_exposure", "lateral_movement_path"], "acceptable_evidence": [{"kind": "topology_model", "minimum_confidence": "medium"}], - "risk": {"severity": "high", "likelihood": "medium", "rationale": "Topology regressions enable data exfiltration."}, + "risk": { + "severity": "high", + "likelihood": "medium", + "rationale": "Topology regressions enable data exfiltration.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -153,7 +170,11 @@ }, "failure_modes": ["skipped_approval", "invalid_state_transition"], "acceptable_evidence": [{"kind": "model_check", "minimum_confidence": "high"}], - "risk": {"severity": "high", "likelihood": "medium", "rationale": "Skipped approvals enable unreviewed production changes."}, + "risk": { + "severity": "high", + "likelihood": "medium", + "rationale": "Skipped approvals enable unreviewed production changes.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -173,7 +194,11 @@ }, "failure_modes": ["buffer_overflow", "use_after_free"], "acceptable_evidence": [{"kind": "memory_model", "minimum_confidence": "high"}], - "risk": {"severity": "critical", "likelihood": "low", "rationale": "Memory corruption can leak cross-tenant data."}, + "risk": { + "severity": "critical", + "likelihood": "low", + "rationale": "Memory corruption can leak cross-tenant data.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -193,7 +218,11 @@ }, "failure_modes": ["self_approval", "gate_removal", "bot_bypass"], "acceptable_evidence": [{"kind": "policy_check", "minimum_confidence": "high"}], - "risk": {"severity": "critical", "likelihood": "high", "rationale": "Agents must not weaken their own verification gates."}, + "risk": { + "severity": "critical", + "likelihood": "high", + "rationale": "Agents must not weaken their own verification gates.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -213,7 +242,11 @@ }, "failure_modes": ["secret_in_untrusted_context", "workflow_permission_escalation"], "acceptable_evidence": [{"kind": "smt_counterexample", "minimum_confidence": "medium"}], - "risk": {"severity": "critical", "likelihood": "medium", "rationale": "Secret exposure in CI is a common agent regression."}, + "risk": { + "severity": "critical", + "likelihood": "medium", + "rationale": "Secret exposure in CI is a common agent regression.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -232,7 +265,11 @@ }, "failure_modes": ["cross_account_trust", "resource_policy_wildcard"], "acceptable_evidence": [{"kind": "policy_check", "minimum_confidence": "high"}], - "risk": {"severity": "high", "likelihood": "medium", "rationale": "Cross-account leaks expand blast radius."}, + "risk": { + "severity": "high", + "likelihood": "medium", + "rationale": "Cross-account leaks expand blast radius.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -270,7 +307,11 @@ }, "failure_modes": ["rollback_bypass", "invalid_state_transition"], "acceptable_evidence": [{"kind": "model_check", "minimum_confidence": "medium"}], - "risk": {"severity": "high", "likelihood": "low", "rationale": "Unsafe rollbacks can reintroduce known failures."}, + "risk": { + "severity": "high", + "likelihood": "low", + "rationale": "Unsafe rollbacks can reintroduce known failures.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -308,7 +349,11 @@ }, "failure_modes": ["privilege_escalation", "self_approval"], "acceptable_evidence": [{"kind": "proof", "minimum_confidence": "high"}], - "risk": {"severity": "critical", "likelihood": "medium", "rationale": "Authority escalation enables autonomous harm."}, + "risk": { + "severity": "critical", + "likelihood": "medium", + "rationale": "Authority escalation enables autonomous harm.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -327,7 +372,11 @@ }, "failure_modes": ["gate_removal", "unsigned_artifact_publish"], "acceptable_evidence": [{"kind": "proof", "minimum_confidence": "medium"}], - "risk": {"severity": "high", "likelihood": "medium", "rationale": "Weakened build gates enable supply-chain regressions."}, + "risk": { + "severity": "high", + "likelihood": "medium", + "rationale": "Weakened build gates enable supply-chain regressions.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -346,7 +395,11 @@ }, "failure_modes": ["privilege_escalation", "policy_default_allow"], "acceptable_evidence": [{"kind": "policy_check", "minimum_confidence": "high"}], - "risk": {"severity": "critical", "likelihood": "medium", "rationale": "Privilege escalation is a primary auth regression class."}, + "risk": { + "severity": "critical", + "likelihood": "medium", + "rationale": "Privilege escalation is a primary auth regression class.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, @@ -365,7 +418,11 @@ }, "failure_modes": ["public_exposure", "lateral_movement_path"], "acceptable_evidence": [{"kind": "topology_model", "minimum_confidence": "medium"}], - "risk": {"severity": "high", "likelihood": "medium", "rationale": "Public egress from sensitive tiers enables exfiltration."}, + "risk": { + "severity": "high", + "likelihood": "medium", + "rationale": "Public egress from sensitive tiers enables exfiltration.", + }, "merge_policy": {"on_pass": "allow", "on_fail": "block", "on_unknown": "require_human_review"}, "provenance": {"source": "ovk-template-library", "canonical": True}, }, diff --git a/scripts/render_bench_badge.py b/scripts/render_bench_badge.py index d6ec933..5c7e23e 100644 --- a/scripts/render_bench_badge.py +++ b/scripts/render_bench_badge.py @@ -75,9 +75,7 @@ def render_summary( "cases_total": summary.get("cases_total", 0), "cases_passed": summary.get("cases_passed", 0), "pass_rate": ( - summary.get("cases_passed", 0) / summary.get("cases_total", 1) - if summary.get("cases_total") - else 0.0 + summary.get("cases_passed", 0) / summary.get("cases_total", 1) if summary.get("cases_total") else 0.0 ), "merge_decision_accuracy": summary.get("merge_decision_accuracy"), "status_accuracy": summary.get("status_accuracy"), diff --git a/scripts/render_pilot_metrics.py b/scripts/render_pilot_metrics.py index eabddcb..d99b4f9 100644 --- a/scripts/render_pilot_metrics.py +++ b/scripts/render_pilot_metrics.py @@ -152,7 +152,9 @@ def parse_args() -> argparse.Namespace: default=EXTERNAL_PILOTS_REGISTRY_PATH, help="External pilots registry JSON (default: docs/benchmarks/external-pilots-registry.json)", ) - parser.add_argument("--output", type=Path, default=ADOPTION_SUMMARY_PATH, help="Output path for adoption-summary.json") + parser.add_argument( + "--output", type=Path, default=ADOPTION_SUMMARY_PATH, help="Output path for adoption-summary.json" + ) parser.add_argument( "--verified-source-sha", default=None, diff --git a/scripts/run_formalpr_holdout.py b/scripts/run_formalpr_holdout.py index b554f5a..19f6ab3 100644 --- a/scripts/run_formalpr_holdout.py +++ b/scripts/run_formalpr_holdout.py @@ -76,9 +76,7 @@ def _walk_keys(value: Any, *, path: str = "$") -> list[tuple[str, str]]: for key, child in value.items(): key_text = str(key) lowered = key_text.lower() - if key_text not in _ALLOWED_META_KEYS and any( - fragment in lowered for fragment in FORBIDDEN_KEY_FRAGMENTS - ): + if key_text not in _ALLOWED_META_KEYS and any(fragment in lowered for fragment in FORBIDDEN_KEY_FRAGMENTS): findings.append((f"{path}.{key_text}", key_text)) findings.extend(_walk_keys(child, path=f"{path}.{key_text}")) elif isinstance(value, list): diff --git a/scripts/sync_real_diff_cases.py b/scripts/sync_real_diff_cases.py index 5fb56df..4807d5c 100644 --- a/scripts/sync_real_diff_cases.py +++ b/scripts/sync_real_diff_cases.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Generate benchmarks/formal_pr_bench/real_diff_cases.json from manifest.""" + from __future__ import annotations import json diff --git a/scripts/write_artifact_manifest.py b/scripts/write_artifact_manifest.py index 2143abc..7052fa3 100644 --- a/scripts/write_artifact_manifest.py +++ b/scripts/write_artifact_manifest.py @@ -13,7 +13,9 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Write an OVK artifact manifest") parser.add_argument("--artifact", action="append", default=[], help="Artifact path to include") - parser.add_argument("--kind", action="append", default=[], help="Optional artifact kind, aligned with --artifact order") + parser.add_argument( + "--kind", action="append", default=[], help="Optional artifact kind, aligned with --artifact order" + ) parser.add_argument("--root", type=Path, default=Path("."), help="Root used for relative paths") parser.add_argument("--output", type=Path, default=Path("ovk-artifact-manifest.json")) return parser.parse_args() @@ -25,8 +27,7 @@ def main() -> int: raise SystemExit("--kind must be supplied once per --artifact when used") kinds = args.kind or ["artifact"] * len(args.artifact) entries = [ - artifact_entry(Path(path), kind=kind, root=args.root) - for path, kind in zip(args.artifact, kinds, strict=True) + artifact_entry(Path(path), kind=kind, root=args.root) for path, kind in zip(args.artifact, kinds, strict=True) ] manifest = build_artifact_manifest(entries) args.output.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") diff --git a/tests/test_adapter_isolation_r2_pr8.py b/tests/test_adapter_isolation_r2_pr8.py index c4f8d84..8d5cc5f 100644 --- a/tests/test_adapter_isolation_r2_pr8.py +++ b/tests/test_adapter_isolation_r2_pr8.py @@ -4,11 +4,10 @@ import json from pathlib import Path -from unittest.mock import patch from ovk.adapters.authorization.z3_adapter import Z3NativeAuthorizationAdapter from ovk.adapters.self_protection.opa_adapter import OpaNativeSelfProtectionAdapter -from ovk.core.execution_budget import LocalSubprocessWorker, WorkerResult +from ovk.core.execution_budget import WorkerResult from ovk.core.execution_models import ExecutionBudget @@ -60,7 +59,16 @@ def test_worker_timeout_enforced_for_z3_not_post_hoc() -> None: compiled = adapter.compile(obligation, routing) class SlowWorker: - def run(self, command, *, cwd, env=None, timeout_seconds=30.0, max_stdout_bytes=1_000_000, max_stderr_bytes=1_000_000): + def run( + self, + command, + *, + cwd, + env=None, + timeout_seconds=30.0, + max_stdout_bytes=1_000_000, + max_stderr_bytes=1_000_000, + ): return WorkerResult( exit_code=None, timed_out=True, diff --git a/tests/test_adversarial_control_plane.py b/tests/test_adversarial_control_plane.py index 72a1ad9..02bd0cf 100644 --- a/tests/test_adversarial_control_plane.py +++ b/tests/test_adversarial_control_plane.py @@ -197,9 +197,7 @@ def test_malformed_backend_output_maps_to_error_or_invalid() -> None: config=RoutingConfig(prefer_deterministic=True, max_selected_backends=1, accept_partial_primary=True), policy={"budget": {"allowed_backends": ["authorization-deterministic"]}}, ) - record = BackendControlPlane(use_hardened_cache=False).execute( - obligation, routing, registry=registry, cache=None - ) + record = BackendControlPlane(use_hardened_cache=False).execute(obligation, routing, registry=registry, cache=None) assert record.results assert record.results[0].status in {VerificationStatus.UNKNOWN, VerificationStatus.ERROR} assert record.attempts[0].termination in {"invalid_output", "completed", "tool_error"} @@ -229,9 +227,7 @@ def test_incomplete_abstraction_cannot_allow_under_strict() -> None: ), policy={"budget": {"allowed_backends": ["authorization-deterministic"]}}, ) - record = BackendControlPlane(use_hardened_cache=False).execute( - obligation, routing, registry=registry, cache=None - ) + record = BackendControlPlane(use_hardened_cache=False).execute(obligation, routing, registry=registry, cache=None) evidence = execution_record_to_evidence(record, routing_enforced=True) assert evidence.decision["merge_recommendation"] != "allow" @@ -266,9 +262,7 @@ def test_backend_timeout_never_deterministic_pass_fallback() -> None: ] } ) - record = BackendControlPlane(use_hardened_cache=False).execute( - obligation, routing, registry=registry, cache=None - ) + record = BackendControlPlane(use_hardened_cache=False).execute(obligation, routing, registry=registry, cache=None) assert record.aggregate_status != VerificationStatus.PASS assert record.merge_recommendation != MergeRecommendation.ALLOW assert record.attempts[0].termination == "timeout" @@ -311,9 +305,7 @@ def test_render_and_attestation_expose_enforced_fields() -> None: config=RoutingConfig(prefer_deterministic=True, max_selected_backends=1), policy={"budget": {"allowed_backends": ["authorization-deterministic"]}}, ) - record = BackendControlPlane(use_hardened_cache=False).execute( - obligation, routing, registry=registry, cache=None - ) + record = BackendControlPlane(use_hardened_cache=False).execute(obligation, routing, registry=registry, cache=None) evidence = execution_record_to_evidence(record, routing_enforced=True, schema_version="ovk.evidence.v2") markdown = render_evidence_markdown(evidence) assert "Compiler:" in markdown diff --git a/tests/test_auth_obligation_cli.py b/tests/test_auth_obligation_cli.py index a1a3143..7fe9520 100644 --- a/tests/test_auth_obligation_cli.py +++ b/tests/test_auth_obligation_cli.py @@ -99,4 +99,3 @@ def test_auth_obligation_cli_writes_quality_output(tmp_path: Path) -> None: payload = json.loads(quality.read_text(encoding="utf-8")) assert payload["schema_version"] == "ovk.evidence_quality.v1" assert payload["passed"] is True - diff --git a/tests/test_authorization_compilers.py b/tests/test_authorization_compilers.py index 3613a18..44c22eb 100644 --- a/tests/test_authorization_compilers.py +++ b/tests/test_authorization_compilers.py @@ -161,7 +161,10 @@ def admin(): def test_acceptance_corpus_meets_program_targets() -> None: cases = build_corpus(meet_targets=True) - by_framework = {"fastapi": {"pass": 0, "fail": 0, "incomplete": 0}, "express": {"pass": 0, "fail": 0, "incomplete": 0}} + by_framework = { + "fastapi": {"pass": 0, "fail": 0, "incomplete": 0}, + "express": {"pass": 0, "fail": 0, "incomplete": 0}, + } for case in cases: classified = classify_case(case) assert classified == case.category, f"{case.case_id} expected {case.category} got {classified}" diff --git a/tests/test_authorization_enforcement.py b/tests/test_authorization_enforcement.py index 1cace6a..a53164b 100644 --- a/tests/test_authorization_enforcement.py +++ b/tests/test_authorization_enforcement.py @@ -62,7 +62,9 @@ def test_policy_changes_selected_backend_execution() -> None: obligation, registry, context=context, - config=RoutingConfig(prefer_deterministic=True, max_selected_backends=1, enforced_lanes=frozenset({"authorization"})), + config=RoutingConfig( + prefer_deterministic=True, max_selected_backends=1, enforced_lanes=frozenset({"authorization"}) + ), policy=det_policy, ) assert [item.backend for item in routing.selected] == ["authorization-deterministic"] @@ -154,7 +156,9 @@ def test_authorization_disagreement_blocks() -> None: outcome = aggregate_fail_dominant_v1( obligation_id="obl", selected=[ - BackendSelection(backend="z3-native", reason="p", expected_guarantee="smt_refutation_search", required=True), + BackendSelection( + backend="z3-native", reason="p", expected_guarantee="smt_refutation_search", required=True + ), BackendSelection( backend="authorization-deterministic", reason="c", diff --git a/tests/test_authorization_input_schema.py b/tests/test_authorization_input_schema.py index c2ae770..a334cd4 100644 --- a/tests/test_authorization_input_schema.py +++ b/tests/test_authorization_input_schema.py @@ -23,14 +23,10 @@ def test_authorization_schema_accepts_valid_protected_fixture() -> None: def test_authorization_schema_rejects_missing_routes_fixture() -> None: - errors = list( - validator().iter_errors(load_json("examples/auth_regression/input_malformed_missing_routes.json")) - ) + errors = list(validator().iter_errors(load_json("examples/auth_regression/input_malformed_missing_routes.json"))) assert errors def test_authorization_schema_rejects_bad_witness_fixture() -> None: - errors = list( - validator().iter_errors(load_json("examples/auth_regression/input_malformed_bad_witness.json")) - ) + errors = list(validator().iter_errors(load_json("examples/auth_regression/input_malformed_bad_witness.json"))) assert errors diff --git a/tests/test_backends.py b/tests/test_backends.py index 7e9bbb1..ccc28bc 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -8,8 +8,12 @@ def test_cedar_backend_pass_and_fail() -> None: - passed = evaluate_cedar_policy(json.loads(Path("examples/backends/cedar_pass.json").read_text(encoding="utf-8")), repo="r", head_sha="sha") - failed = evaluate_cedar_policy(json.loads(Path("examples/backends/cedar_fail.json").read_text(encoding="utf-8")), repo="r", head_sha="sha") + passed = evaluate_cedar_policy( + json.loads(Path("examples/backends/cedar_pass.json").read_text(encoding="utf-8")), repo="r", head_sha="sha" + ) + failed = evaluate_cedar_policy( + json.loads(Path("examples/backends/cedar_fail.json").read_text(encoding="utf-8")), repo="r", head_sha="sha" + ) assert passed.backend_claims[0].status.value == "pass" assert failed.backend_claims[0].status.value == "fail" diff --git a/tests/test_cache_worker_control_plane.py b/tests/test_cache_worker_control_plane.py index 78712c6..6ef0510 100644 --- a/tests/test_cache_worker_control_plane.py +++ b/tests/test_cache_worker_control_plane.py @@ -78,12 +78,8 @@ def test_control_plane_cache_hit_and_subject_mismatch(tmp_path: Path) -> None: def test_policy_digest_mismatch_does_not_reuse_cache(tmp_path: Path) -> None: data = json.loads(Path("examples/auth_regression/input_admin_protected.json").read_text(encoding="utf-8")) registry = build_authorization_registry() - obligation_a = compile_authorization_obligation( - data, repo="r", head_sha="h", policy_digest="policy-a" - ) - obligation_b = compile_authorization_obligation( - data, repo="r", head_sha="h", policy_digest="policy-b" - ) + obligation_a = compile_authorization_obligation(data, repo="r", head_sha="h", policy_digest="policy-a") + obligation_b = compile_authorization_obligation(data, repo="r", head_sha="h", policy_digest="policy-b") assert obligation_a.policy_digest != obligation_b.policy_digest budget = _budget() cache = ControlPlaneResultCache(HardenedResultCache(tmp_path)) diff --git a/tests/test_change_detection.py b/tests/test_change_detection.py index 5392f8c..9d8c730 100644 --- a/tests/test_change_detection.py +++ b/tests/test_change_detection.py @@ -17,10 +17,12 @@ def test_infrastructure_change_selects_infra_intent() -> None: def test_detect_change_surfaces_groups_files() -> None: - surfaces = detect_change_surfaces([ - ".github/workflows/verify.yml", - "src/middleware/auth.py", - "main.tf", - ]) + surfaces = detect_change_surfaces( + [ + ".github/workflows/verify.yml", + "src/middleware/auth.py", + "main.tf", + ] + ) domains = {surface.domain for surface in surfaces} assert {"ci_cd", "authorization", "infrastructure"}.issubset(domains) diff --git a/tests/test_compiler_runtime_wiring.py b/tests/test_compiler_runtime_wiring.py index da5f705..feca7f0 100644 --- a/tests/test_compiler_runtime_wiring.py +++ b/tests/test_compiler_runtime_wiring.py @@ -218,7 +218,9 @@ def test_deployment_github_environments_compiler() -> None: obligation, registry, context=ExecutionContext(subject=obligation.subject, budget=budget, policy_digest="p"), - config=RoutingConfig(prefer_deterministic=True, max_selected_backends=1, enforced_lanes=frozenset({"deployment"})), + config=RoutingConfig( + prefer_deterministic=True, max_selected_backends=1, enforced_lanes=frozenset({"deployment"}) + ), ) record = BackendControlPlane().execute(obligation, routing, registry=registry) assert record.results diff --git a/tests/test_decision.py b/tests/test_decision.py index db8fb8e..f5e1650 100644 --- a/tests/test_decision.py +++ b/tests/test_decision.py @@ -38,10 +38,7 @@ def test_unknown_requires_human_review_in_enforce_mode() -> None: def test_unknown_blocks_when_default_on_unknown_is_block() -> None: - assert ( - decide(make_bundle("unknown"), enforce=True, default_on_unknown="block") - == MergeRecommendation.BLOCK - ) + assert decide(make_bundle("unknown"), enforce=True, default_on_unknown="block") == MergeRecommendation.BLOCK def test_unknown_allows_with_warning_when_configured() -> None: diff --git a/tests/test_deterministic_worker_r2_pr7.py b/tests/test_deterministic_worker_r2_pr7.py index 1987837..cf9a14b 100644 --- a/tests/test_deterministic_worker_r2_pr7.py +++ b/tests/test_deterministic_worker_r2_pr7.py @@ -5,7 +5,6 @@ import json from pathlib import Path -import pytest from ovk.adapters.authorization.deterministic_adapter import AuthorizationDeterministicAdapter from ovk.core.backend_control_plane import BackendControlPlane @@ -18,13 +17,17 @@ def test_deterministic_adapter_requires_worker() -> None: adapter = AuthorizationDeterministicAdapter() obligation = adapter.compile( - __import__("ovk.core.authorization_compiler", fromlist=["compile_authorization_obligation"]).compile_authorization_obligation( + __import__( + "ovk.core.authorization_compiler", fromlist=["compile_authorization_obligation"] + ).compile_authorization_obligation( json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")), repo="r", head_sha="h", ), __import__("ovk.core.routing_pipeline", fromlist=["route_compiled_obligation"]).route_compiled_obligation( - __import__("ovk.core.authorization_compiler", fromlist=["compile_authorization_obligation"]).compile_authorization_obligation( + __import__( + "ovk.core.authorization_compiler", fromlist=["compile_authorization_obligation"] + ).compile_authorization_obligation( json.loads(Path("examples/auth_regression/input_admin_bypass.json").read_text(encoding="utf-8")), repo="r", head_sha="h", diff --git a/tests/test_emit_github_check.py b/tests/test_emit_github_check.py index 9e35903..3be9f8a 100644 --- a/tests/test_emit_github_check.py +++ b/tests/test_emit_github_check.py @@ -50,17 +50,20 @@ def test_emit_github_check_posts_check_run(tmp_path: Path, monkeypatch) -> None: response.__enter__ = MagicMock(return_value=response) response.__exit__ = MagicMock(return_value=False) - with patch("sys.argv", [ - "emit_github_check.py", - "--evidence", - str(evidence), - "--markdown", - str(markdown), - "--repo", - "owner/repo", - "--head-sha", - "abc123", - ]): + with patch( + "sys.argv", + [ + "emit_github_check.py", + "--evidence", + str(evidence), + "--markdown", + str(markdown), + "--repo", + "owner/repo", + "--head-sha", + "abc123", + ], + ): with patch("urllib.request.urlopen", return_value=response) as urlopen: assert main() == 0 request = urlopen.call_args.args[0] @@ -75,15 +78,18 @@ def test_emit_github_check_api_failure_returns_one(tmp_path: Path, monkeypatch) _write_evidence(evidence) monkeypatch.setenv("GITHUB_TOKEN", "test-token") - with patch("sys.argv", [ - "emit_github_check.py", - "--evidence", - str(evidence), - "--repo", - "owner/repo", - "--head-sha", - "abc123", - ]): + with patch( + "sys.argv", + [ + "emit_github_check.py", + "--evidence", + str(evidence), + "--repo", + "owner/repo", + "--head-sha", + "abc123", + ], + ): with patch("scripts.emit_github_check._post_check_run", return_value=False): assert main() == 1 diff --git a/tests/test_infra_graph.py b/tests/test_infra_graph.py index 3a978e8..2c20dca 100644 --- a/tests/test_infra_graph.py +++ b/tests/test_infra_graph.py @@ -29,6 +29,8 @@ def test_graph_disconnected_sensitive_resource_allows() -> None: def test_empty_graph_becomes_invalid_infra_input() -> None: - evidence = evaluate_infra_exposure(graph_to_infra_input({"nodes": [], "edges": []}), repo="example/repo", head_sha="abc") + evidence = evaluate_infra_exposure( + graph_to_infra_input({"nodes": [], "edges": []}), repo="example/repo", head_sha="abc" + ) assert evidence.backend_claims[0].status.value == "unknown" assert evidence.decision["merge_recommendation"] == "require_human_review" diff --git a/tests/test_ingest_external_pilot_metrics.py b/tests/test_ingest_external_pilot_metrics.py index dec18fb..0544086 100644 --- a/tests/test_ingest_external_pilot_metrics.py +++ b/tests/test_ingest_external_pilot_metrics.py @@ -141,7 +141,15 @@ def test_validate_registry_rejects_invalid_row(tmp_path: Path) -> None: registry = { "schema_version": "ovk.external_pilots_registry.v1", "updated_at": "2026-06-10T00:00:00Z", - "external_pilots": [{"repository": "bad/repo", "status": "unknown", "check_types": ["ci_secrets"], "strict_enabled": False, "ovk_version_pin": "1.2.0"}], + "external_pilots": [ + { + "repository": "bad/repo", + "status": "unknown", + "check_types": ["ci_secrets"], + "strict_enabled": False, + "ovk_version_pin": "1.2.0", + } + ], } with pytest.raises(ValueError, match="external pilots registry failed schema validation"): validate_registry(registry) diff --git a/tests/test_material_reference_sizes.py b/tests/test_material_reference_sizes.py index 1db580c..127ca81 100644 --- a/tests/test_material_reference_sizes.py +++ b/tests/test_material_reference_sizes.py @@ -74,6 +74,7 @@ def test_shadow_and_bridge_material_sizes_bind_payload() -> None: assert bridged.size_bytes == len(canonical_material_bytes(data)) assert bridged.size_bytes != len(bridged.sha256) + def test_ci_secrets_legacy_material_size_binds_input_payload() -> None: data = { "trust_context": "untrusted_fork_pr", @@ -96,4 +97,3 @@ def test_ci_secrets_legacy_material_size_binds_input_payload() -> None: reference = obligation.materials[0] assert reference.size_bytes == len(canonical_material_bytes(data)) assert reference.size_bytes != len(reference.sha256) - diff --git a/tests/test_native_backends.py b/tests/test_native_backends.py index 39394b8..974e3e2 100644 --- a/tests/test_native_backends.py +++ b/tests/test_native_backends.py @@ -20,10 +20,7 @@ def _assert_backend(backend: str) -> None: + (f", detail={result.detail}" if getattr(result, "detail", None) else "") ) if result.binary_present and backend in TIER1_NATIVE_EXECUTION_BACKENDS: - assert result.used_native_binary, ( - f"{backend} detected {result.binary_name} " - "but did not report native use" - ) + assert result.used_native_binary, f"{backend} detected {result.binary_name} but did not report native use" @requires_native_ci diff --git a/tests/test_opa_infra_domain.py b/tests/test_opa_infra_domain.py index 12fe204..5284420 100644 --- a/tests/test_opa_infra_domain.py +++ b/tests/test_opa_infra_domain.py @@ -5,14 +5,18 @@ def test_opa_infra_domain_pack_blocks_public_sensitive_resource() -> None: - payload = json.loads(Path("examples/infrastructure_exposure/input_public_sensitive_resource.json").read_text(encoding="utf-8")) + payload = json.loads( + Path("examples/infrastructure_exposure/input_public_sensitive_resource.json").read_text(encoding="utf-8") + ) evidence = evaluate_infra_exposure_opa(payload, repo="r", head_sha="sha") assert evidence.backend_claims[0].status.value == "fail" assert evidence.counterexamples[0]["failure_mode"] == "sensitive_resource_publicly_exposed" def test_opa_infra_domain_pack_allows_private_resource() -> None: - payload = json.loads(Path("examples/infrastructure_exposure/input_private_sensitive_resource.json").read_text(encoding="utf-8")) + payload = json.loads( + Path("examples/infrastructure_exposure/input_private_sensitive_resource.json").read_text(encoding="utf-8") + ) evidence = evaluate_infra_exposure_opa(payload, repo="r", head_sha="sha") assert evidence.backend_claims[0].status.value == "pass" diff --git a/tests/test_phase3_trust.py b/tests/test_phase3_trust.py index 0d9d9ac..0450feb 100644 --- a/tests/test_phase3_trust.py +++ b/tests/test_phase3_trust.py @@ -12,7 +12,9 @@ def test_adversarial_sha_mismatch_fails_quality_gate() -> None: - bundle = EvidenceBundle.model_validate(read_json_file(Path("examples/evidence_quality/adversarial_sha_mismatch.json"))) + bundle = EvidenceBundle.model_validate( + read_json_file(Path("examples/evidence_quality/adversarial_sha_mismatch.json")) + ) report = build_evidence_quality_report(bundle) assert report.passed is False messages = {issue.message for issue in report.issues} diff --git a/tests/test_phase7_v1.py b/tests/test_phase7_v1.py index 3ff427f..4a71ee4 100644 --- a/tests/test_phase7_v1.py +++ b/tests/test_phase7_v1.py @@ -50,8 +50,7 @@ def test_bench_cli_writes_leaderboard(tmp_path: Path) -> None: def test_v1_readiness_checklist() -> None: metadata = release_metadata() backends = { - manifest.get("tool", {}).get("name") - for manifest in CapabilityRegistry.from_directory(Path("adapters")).all() + manifest.get("tool", {}).get("name") for manifest in CapabilityRegistry.from_directory(Path("adapters")).all() } required_backends = {"opa", "z3", "cedar", "tla+", "kani", "dafny", "verus", "lean", "cbmc", "alloy"} assert required_backends.issubset(backends) diff --git a/tests/test_policy_loading_fail_closed.py b/tests/test_policy_loading_fail_closed.py index 96f3865..e1f2705 100644 --- a/tests/test_policy_loading_fail_closed.py +++ b/tests/test_policy_loading_fail_closed.py @@ -15,9 +15,7 @@ def test_malformed_policy_yaml_is_rejected(tmp_path: Path) -> None: def test_schema_invalid_policy_is_rejected(tmp_path: Path) -> None: config = tmp_path / "config.yml" config.write_text( - "schema_version: ovk.config.v1\n" - "mode: strict\n" - "default_on_unknown: allow\n", + "schema_version: ovk.config.v1\nmode: strict\ndefault_on_unknown: allow\n", encoding="utf-8", ) with pytest.raises(ValueError, match="failed schema validation"): diff --git a/tests/test_render_bundle.py b/tests/test_render_bundle.py index f446690..dd161cd 100644 --- a/tests/test_render_bundle.py +++ b/tests/test_render_bundle.py @@ -7,18 +7,14 @@ def test_bundle_blocks_when_evidence_fails() -> None: - data = json.loads( - Path("examples/no_agent_self_approval/input_gate_removed.json").read_text(encoding="utf-8") - ) + data = json.loads(Path("examples/no_agent_self_approval/input_gate_removed.json").read_text(encoding="utf-8")) evidence = evaluate_self_protection(data, repo="example/repo", head_sha="abc") bundle = make_bundle([evidence]) assert bundle.decision["merge_recommendation"] == "block" def test_renderer_includes_guarantee_status_and_counterexample() -> None: - data = json.loads( - Path("examples/no_agent_self_approval/input_gate_removed.json").read_text(encoding="utf-8") - ) + data = json.loads(Path("examples/no_agent_self_approval/input_gate_removed.json").read_text(encoding="utf-8")) evidence = evaluate_self_protection(data, repo="example/repo", head_sha="abc") bundle = make_bundle([evidence]) rendered = render_bundle_markdown(bundle) diff --git a/tests/test_render_pilot_metrics.py b/tests/test_render_pilot_metrics.py index 185c970..124b87c 100644 --- a/tests/test_render_pilot_metrics.py +++ b/tests/test_render_pilot_metrics.py @@ -100,8 +100,26 @@ def test_render_merge_preserves_registry_on_rerender(tmp_path: Path) -> None: def test_merge_external_pilots_registry_wins_on_conflict() -> None: merged = merge_external_pilots( - [{"repository": "org/repo", "status": "advisory", "check_types": ["ci_secrets"], "strict_enabled": False, "ovk_version_pin": "1.2.0", "prs_evaluated": 5}], - [{"repository": "org/repo", "status": "recruiting", "check_types": ["ci_secrets"], "strict_enabled": False, "ovk_version_pin": "1.2.0", "prs_evaluated": 1}], + [ + { + "repository": "org/repo", + "status": "advisory", + "check_types": ["ci_secrets"], + "strict_enabled": False, + "ovk_version_pin": "1.2.0", + "prs_evaluated": 5, + } + ], + [ + { + "repository": "org/repo", + "status": "recruiting", + "check_types": ["ci_secrets"], + "strict_enabled": False, + "ovk_version_pin": "1.2.0", + "prs_evaluated": 1, + } + ], ) assert len(merged) == 1 assert merged[0]["status"] == "advisory" diff --git a/tests/test_repair_loop.py b/tests/test_repair_loop.py index 31eb478..bc1b46c 100644 --- a/tests/test_repair_loop.py +++ b/tests/test_repair_loop.py @@ -14,9 +14,7 @@ def test_ci_secrets_repair_loop_failing_diff_blocks() -> None: result = run_check(diff_text=diff_text, repo="test/repo", head_sha="fail", use_cache=False) assert result.bundle.decision.get("merge_recommendation") == "block" counterexamples = [ - counterexample - for evidence in result.bundle.evidence - for counterexample in evidence.counterexamples + counterexample for evidence in result.bundle.evidence for counterexample in evidence.counterexamples ] assert counterexamples diff --git a/tests/test_repo_memory_trust.py b/tests/test_repo_memory_trust.py index 4baf2ce..f5faf60 100644 --- a/tests/test_repo_memory_trust.py +++ b/tests/test_repo_memory_trust.py @@ -10,9 +10,7 @@ def _write_run(path: Path, status: str) -> None: { "backend_outcomes": [{"backend": "z3", "status": status}], "lanes": ["no-admin-route-bypass"], - "decision": { - "merge_recommendation": "block" if status == "fail" else "require_human_review" - }, + "decision": {"merge_recommendation": "block" if status == "fail" else "require_human_review"}, } ), encoding="utf-8", diff --git a/tests/test_result_cache_semantics.py b/tests/test_result_cache_semantics.py index 250c0cb..06f132b 100644 --- a/tests/test_result_cache_semantics.py +++ b/tests/test_result_cache_semantics.py @@ -33,8 +33,22 @@ def fake_evaluate_lane(lane, data, *, repo, head_sha, base_sha, input_format, po monkeypatch.setattr(adapter_runtime, "evaluate_lane", fake_evaluate_lane) raw_input = {"resource_changes": []} - native = [{"lane": "infrastructure", "intent_id": "no-public-sensitive-resource", "input": raw_input, "input_format": "infra"}] - terraform = [{"lane": "infrastructure", "intent_id": "no-public-sensitive-resource", "input": raw_input, "input_format": "terraform"}] + native = [ + { + "lane": "infrastructure", + "intent_id": "no-public-sensitive-resource", + "input": raw_input, + "input_format": "infra", + } + ] + terraform = [ + { + "lane": "infrastructure", + "intent_id": "no-public-sensitive-resource", + "input": raw_input, + "input_format": "terraform", + } + ] adapter_runtime.execute_obligations( native, diff --git a/tests/test_routing_pipeline_r2_pr6.py b/tests/test_routing_pipeline_r2_pr6.py index 0b1d94f..8a0bbb7 100644 --- a/tests/test_routing_pipeline_r2_pr6.py +++ b/tests/test_routing_pipeline_r2_pr6.py @@ -75,9 +75,7 @@ def test_kernel_mcp_and_evidence_share_routing_id() -> None: ) assert evidence_items[0].routing_id == expected_routing_id routing_artifacts = [ - artifact - for artifact in evidence_items[0].generated_artifacts - if artifact.get("kind") == "backend_routing" + artifact for artifact in evidence_items[0].generated_artifacts if artifact.get("kind") == "backend_routing" ] assert routing_artifacts assert routing_artifacts[0]["routing_id"] == expected_routing_id @@ -135,7 +133,6 @@ def test_kernel_uses_authoritative_routing_for_obligations() -> None: auth_routing = next( item for item in result.routing - if item.get("obligation_id") == auth_evidence.obligation_id - or item.get("intent_id") == "no-admin-route-bypass" + if item.get("obligation_id") == auth_evidence.obligation_id or item.get("intent_id") == "no-admin-route-bypass" ) assert auth_routing["routing_id"] == auth_evidence.routing_id diff --git a/tests/test_runtime_cache_regimes.py b/tests/test_runtime_cache_regimes.py index 20f860e..5573551 100644 --- a/tests/test_runtime_cache_regimes.py +++ b/tests/test_runtime_cache_regimes.py @@ -5,9 +5,7 @@ def _auth_input() -> dict: - return json.loads( - Path("examples/auth_regression/input_admin_protected.json").read_text(encoding="utf-8") - ) + return json.loads(Path("examples/auth_regression/input_admin_protected.json").read_text(encoding="utf-8")) def test_enforced_execution_does_not_reuse_legacy_flat_cache(tmp_path: Path) -> None: diff --git a/tests/test_scripts_sprint2.py b/tests/test_scripts_sprint2.py index 692e735..4ecf5e7 100644 --- a/tests/test_scripts_sprint2.py +++ b/tests/test_scripts_sprint2.py @@ -9,15 +9,7 @@ def test_normalize_required_checks_script(tmp_path: Path, monkeypatch) -> None: input_path = tmp_path / "branch_protection.json" output_path = tmp_path / "required_checks.json" input_path.write_text( - json.dumps( - { - "after_branch_protection": { - "required_status_checks": { - "contexts": ["unit-tests", "ovk-verify"] - } - } - } - ), + json.dumps({"after_branch_protection": {"required_status_checks": {"contexts": ["unit-tests", "ovk-verify"]}}}), encoding="utf-8", ) monkeypatch.setattr( diff --git a/tests/test_self_protection_enforcement.py b/tests/test_self_protection_enforcement.py index 7406838..265d3ba 100644 --- a/tests/test_self_protection_enforcement.py +++ b/tests/test_self_protection_enforcement.py @@ -80,13 +80,18 @@ def test_enforced_self_protection_blocks_gate_removal() -> None: def test_untrusted_metadata_cannot_allow(monkeypatch: pytest.MonkeyPatch) -> None: - data = _load_example("input_gate_preserved.json") if Path("examples/no_agent_self_approval/input_gate_preserved.json").exists() or Path("ovk/package_data/examples/no_agent_self_approval/input_gate_preserved.json").exists() else { - "actor": {"type": "ai_agent", "id": "bot"}, - "changed_files": ["README.md"], - "before": {"required_checks": ["ovk-verify"]}, - "after": {"required_checks": ["ovk-verify"]}, - "ovk_gate_name": "ovk-verify", - } + data = ( + _load_example("input_gate_preserved.json") + if Path("examples/no_agent_self_approval/input_gate_preserved.json").exists() + or Path("ovk/package_data/examples/no_agent_self_approval/input_gate_preserved.json").exists() + else { + "actor": {"type": "ai_agent", "id": "bot"}, + "changed_files": ["README.md"], + "before": {"required_checks": ["ovk-verify"]}, + "after": {"required_checks": ["ovk-verify"]}, + "ovk_gate_name": "ovk-verify", + } + ) if isinstance(data, str): data = _load_example("input_gate_preserved.json") evidence_items = execute_obligations( diff --git a/tests/test_self_protection_sprint1.py b/tests/test_self_protection_sprint1.py index 9257e07..9b32a4d 100644 --- a/tests/test_self_protection_sprint1.py +++ b/tests/test_self_protection_sprint1.py @@ -6,9 +6,7 @@ def test_missing_required_check_metadata_returns_unknown() -> None: - data = json.loads( - Path("examples/no_agent_self_approval/input_missing_metadata.json").read_text(encoding="utf-8") - ) + data = json.loads(Path("examples/no_agent_self_approval/input_missing_metadata.json").read_text(encoding="utf-8")) evidence = evaluate_self_protection(data, repo="example/repo", head_sha="abc") assert evidence.backend_claims[0].status.value == "unknown" assert evidence.decision["merge_recommendation"] == "require_human_review" diff --git a/tests/test_shadow_control_plane.py b/tests/test_shadow_control_plane.py index ad3e4fc..8411546 100644 --- a/tests/test_shadow_control_plane.py +++ b/tests/test_shadow_control_plane.py @@ -23,7 +23,9 @@ from ovk.core.shadow_obligation import build_shadow_obligation -def _result(backend: str, status: VerificationStatus, *, guarantee: str = "smt_refutation_search") -> NormalizedBackendResult: +def _result( + backend: str, status: VerificationStatus, *, guarantee: str = "smt_refutation_search" +) -> NormalizedBackendResult: return NormalizedBackendResult( attempt_id=f"att-{backend}", backend=backend, diff --git a/tests/test_sigstore_release.py b/tests/test_sigstore_release.py index 537ca1e..a9470f8 100644 --- a/tests/test_sigstore_release.py +++ b/tests/test_sigstore_release.py @@ -21,27 +21,21 @@ def test_github_certificate_identity_from_workflow_ref() -> None: identity = github_certificate_identity( - workflow_ref=( - "fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.0" - ) + workflow_ref=("fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.0") ) assert identity == ( - "https://github.com/fraware/open-verification-kernel/" - ".github/workflows/publish.yml@refs/tags/v1.2.0" + "https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.0" ) def test_github_certificate_identity_rejects_foreign_repo() -> None: with pytest.raises(ValueError, match="unexpected workflow_ref"): - github_certificate_identity( - workflow_ref="evil/repo/.github/workflows/publish.yml@refs/tags/v1.2.0" - ) + github_certificate_identity(workflow_ref="evil/repo/.github/workflows/publish.yml@refs/tags/v1.2.0") def test_production_tag_identity_matches_release_policy() -> None: assert production_tag_identity("v1.2.0") == ( - "https://github.com/fraware/open-verification-kernel/" - ".github/workflows/publish.yml@refs/tags/v1.2.0" + "https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.0" ) assert production_tag_identity("refs/tags/v1.2.0") == production_tag_identity("v1.2.0") diff --git a/tests/test_source_profile_hardening.py b/tests/test_source_profile_hardening.py index eaeafad..b7b8699 100644 --- a/tests/test_source_profile_hardening.py +++ b/tests/test_source_profile_hardening.py @@ -27,13 +27,7 @@ def test_fastapi_ast_detects_admin_bypass() -> None: "def users():\n" " return []\n" ) - head = ( - "from fastapi import FastAPI\n" - "app = FastAPI()\n" - "@app.get('/admin/users')\n" - "def users():\n" - " return []\n" - ) + head = "from fastapi import FastAPI\napp = FastAPI()\n@app.get('/admin/users')\ndef users():\n return []\n" materials = materials_from_pair(path="app.py", base_source=base, head_source=head) ir = FastApiAstAuthorizationCompiler().compile(materials) assert any("authorization.fastapi.ast_v1" in note for note in ir.warnings) @@ -42,12 +36,7 @@ def test_fastapi_ast_detects_admin_bypass() -> None: def test_fastapi_ast_marks_dynamic_path_unsupported() -> None: source = ( - "from fastapi import FastAPI\n" - "app = FastAPI()\n" - "path = '/x'\n" - "@app.get(path)\n" - "def handler():\n" - " return {}\n" + "from fastapi import FastAPI\napp = FastAPI()\npath = '/x'\n@app.get(path)\ndef handler():\n return {}\n" ) materials = materials_from_pair(path="app.py", base_source=source, head_source=source) ir = FastApiAstAuthorizationCompiler().compile(materials) @@ -113,21 +102,13 @@ def test_kubernetes_controller_selector_edge() -> None: def test_profile_provers_and_bindings() -> None: repo = Path(__file__).resolve().parents[1] assert compiler_binding_for("authorization.fastapi.ast_v1") - fastapi = prove_fastapi_ast_profile( - repo, enforcement_test="tests/test_authorization_enforcement.py" - ) + fastapi = prove_fastapi_ast_profile(repo, enforcement_test="tests/test_authorization_enforcement.py") assert fastapi.as_dict()["strict_eligible"] is True - tf = prove_terraform_recursive_profile( - repo, enforcement_test="tests/test_remaining_lane_enforcement.py" - ) + tf = prove_terraform_recursive_profile(repo, enforcement_test="tests/test_remaining_lane_enforcement.py") assert tf.as_dict()["strict_eligible"] is True - k8s = prove_k8s_controller_profile( - repo, enforcement_test="tests/test_remaining_lane_enforcement.py" - ) + k8s = prove_k8s_controller_profile(repo, enforcement_test="tests/test_remaining_lane_enforcement.py") assert k8s.as_dict()["strict_eligible"] is True - actions = prove_actions_permissions_flow( - repo, enforcement_test="tests/test_remaining_lane_enforcement.py" - ) + actions = prove_actions_permissions_flow(repo, enforcement_test="tests/test_remaining_lane_enforcement.py") assert actions.as_dict()["strict_eligible"] is True diff --git a/tests/test_typed_router.py b/tests/test_typed_router.py index d5daa32..17d6850 100644 --- a/tests/test_typed_router.py +++ b/tests/test_typed_router.py @@ -71,7 +71,12 @@ def _obligation(*, lane: str = "authorization") -> VerificationObligation: abstraction=abstraction, abstraction_digest=compute_abstraction_digest(abstraction), coverage=AbstractionCoverage(status="complete", confidence=1.0, extracted_elements=1), - acceptable_guarantees=["smt_refutation_search", "policy_evaluation", "workflow_secrets_boundary_check", "state_machine_safety"], + acceptable_guarantees=[ + "smt_refutation_search", + "policy_evaluation", + "workflow_secrets_boundary_check", + "state_machine_safety", + ], required_capabilities=[], policy_digest="policy-digest", ) diff --git a/tests/test_verification_cache.py b/tests/test_verification_cache.py index 58dab0b..ec73fba 100644 --- a/tests/test_verification_cache.py +++ b/tests/test_verification_cache.py @@ -46,7 +46,9 @@ def _budget() -> ExecutionBudget: ) -def _obligation(*, repo: str = "acme/api", head_sha: str = "abc", base_sha: str | None = "def") -> VerificationObligation: +def _obligation( + *, repo: str = "acme/api", head_sha: str = "abc", base_sha: str | None = "def" +) -> VerificationObligation: abstraction = {"kind": "test", "input": {"x": 1}} provisional = VerificationObligation( obligation_id="pending", diff --git a/tests/test_z3_obligations.py b/tests/test_z3_obligations.py index 5d9f24c..29e32ef 100644 --- a/tests/test_z3_obligations.py +++ b/tests/test_z3_obligations.py @@ -20,9 +20,7 @@ def test_admin_route_bypass_fixture_emits_counterexample() -> None: counterexamples = find_authorization_counterexamples(data) assert counterexamples assert counterexamples[0]["failure_mode"] == "admin_route_reachable_by_non_admin" - expected = json.loads( - Path("examples/z3_fail/admin_route_bypass.counterexample.json").read_text(encoding="utf-8") - ) + expected = json.loads(Path("examples/z3_fail/admin_route_bypass.counterexample.json").read_text(encoding="utf-8")) assert counterexamples[0]["route"] == expected["route"] assert counterexamples[0]["user_role"] == expected["user_role"] @@ -32,9 +30,7 @@ def test_privilege_escalation_fixture_emits_counterexample() -> None: counterexamples = find_privilege_escalation_counterexamples(data) assert counterexamples assert counterexamples[0]["failure_mode"] == "privilege_escalation" - expected = json.loads( - Path("examples/z3_fail/privilege_escalation.counterexample.json").read_text(encoding="utf-8") - ) + expected = json.loads(Path("examples/z3_fail/privilege_escalation.counterexample.json").read_text(encoding="utf-8")) assert counterexamples[0]["principal"] == expected["principal"] assert counterexamples[0]["gained_role"] == expected["gained_role"] diff --git a/tests/test_z3_validation.py b/tests/test_z3_validation.py index 58b072c..81d8f16 100644 --- a/tests/test_z3_validation.py +++ b/tests/test_z3_validation.py @@ -115,7 +115,4 @@ def test_validated_path_valid_input_does_not_emit_validation_failure() -> None: repo="example/repo", head_sha="abc", ) - assert all( - item.get("failure_mode") != "authorization_abstraction_invalid" - for item in evidence.counterexamples - ) + assert all(item.get("failure_mode") != "authorization_abstraction_invalid" for item in evidence.counterexamples)