From 7df6e391516b3826cac0d2f79a28d288849c2b39 Mon Sep 17 00:00:00 2001 From: Punisheroot <44579963+Punisheroot@users.noreply.github.com> Date: Tue, 4 Aug 2026 12:01:32 +0200 Subject: [PATCH] feat(bench): add sealed holdout corpus protocol --- PROJECT_STATUS.md | 14 +- benchmarks/README.md | 13 +- benchmarks/corpus/router-cache/campaign.json | 8 +- .../router-cache/legacy-offline-campaign.json | 26 + .../router-cache/legacy-offline-manifest.json | 70 ++ benchmarks/corpus/router-cache/manifest.json | 98 +-- .../corpus/router-cache/oracles/README.md | 12 + .../corpus/router-cache/power-plan.json | 32 + benchmarks/corpus/router-cache/schedule.json | 28 + .../docs/ripgrep-crlf-trace-calibration.json | 16 + ...b-case-insensitive-locate-calibration.json | 16 + .../ripgrep-no-ignore-vcs-locate-holdout.json | 16 + .../docs/ripgrep-null-data-trace-holdout.json | 16 + .../router-cache/synthetic-sealed/index.json | 41 + crates/needle-app/src/main.rs | 8 +- crates/needle-app/src/minimal_live_pilot.rs | 31 +- .../src/minimal_live_pilot/protocol.rs | 52 +- .../needle-app/src/worker_live_diagnostic.rs | 12 +- .../needle-bench/src/bin/corpus-preflight.rs | 34 +- crates/needle-bench/src/corpus.rs | 628 ++++++++++++-- crates/needle-bench/src/final_gate.rs | 214 ++++- crates/needle-bench/src/lib.rs | 24 +- crates/needle-bench/src/schedule.rs | 635 +++++++++++++++ crates/needle-bench/src/sealed_oracle.rs | 767 ++++++++++++++++++ .../needle-bench/tests/publication_surface.rs | 67 ++ docs/BENCHMARKING.md | 17 +- 26 files changed, 2717 insertions(+), 178 deletions(-) create mode 100644 benchmarks/corpus/router-cache/legacy-offline-campaign.json create mode 100644 benchmarks/corpus/router-cache/legacy-offline-manifest.json create mode 100644 benchmarks/corpus/router-cache/oracles/README.md create mode 100644 benchmarks/corpus/router-cache/power-plan.json create mode 100644 benchmarks/corpus/router-cache/schedule.json create mode 100644 benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-crlf-trace-calibration.json create mode 100644 benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-glob-case-insensitive-locate-calibration.json create mode 100644 benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-no-ignore-vcs-locate-holdout.json create mode 100644 benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-null-data-trace-holdout.json create mode 100644 benchmarks/corpus/router-cache/synthetic-sealed/index.json create mode 100644 crates/needle-bench/src/schedule.rs create mode 100644 crates/needle-bench/src/sealed_oracle.rs diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index c26311a..6ff4a5d 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -140,7 +140,13 @@ validation. OpenCode, and Antigravity configuration and execution are unsupported. - Linux and macOS lack live platform evidence. - The accepted economic results are calibrations, not a powered corpus or - general savings claim. + general savings claim. The public v4 benchmark manifest is answer-free and + synthetic; the offline protocol and synthetic evaluator fixture are complete, + but real private material, a non-synthetic validated PowerPlan, and a + provider run are absent. App provider execution remains fail-closed until an + isolated executor/broker consumes only ArmLaunch. This does not prove + filesystem ACLs or process isolation; callers must keep private bundles + unmounted and inaccessible to the runner identity. - No public beta, support channel, or compatibility window exists. ## Next milestone @@ -150,8 +156,10 @@ Release readiness requires all of the following: 1. provider-backed calibration of authoritative claim reuse; 2. live end-to-end verified-change validation (`prepare_change` -> `verify_change` / patcher-verifier); 3. a second live platform; -4. a frozen multi-task corpus with independent oracles; -5. a powered paired analysis and confidence interval; +4. a frozen answer-free multi-task corpus bound to maintainer-owned sealed + evaluator material; +5. a validated production PowerPlan, powered paired analysis, and confidence + interval; 6. stable developer packaging and a compatibility policy; 7. a dedicated security and publication review. diff --git a/benchmarks/README.md b/benchmarks/README.md index 72bd57f..535426c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -11,7 +11,8 @@ savings claim. ```text benchmarks/ - corpus/router-cache/ frozen tasks, oracles, campaign, and cost model + corpus/router-cache/ answer-free manifest, schedule, synthetic plan, campaign, and cost model + oracles/ legacy synthetic evaluator fixtures only (not public handles) fixtures/ structured protocol and worker fixtures results/historical/ accepted observations from an earlier product boundary results/live/ accepted provider-backed current-path calibration @@ -65,4 +66,12 @@ are excluded from published benchmark evidence and do not support benchmark claims. The evidence index records observations; it does not approve or schedule -provider runs. +provider runs. Public corpus material is answer-free and synthetic. The +evaluator-owned sealed oracle index and its bytes are supplied out of band by +maintainers; no external bundle path is present in public manifests or arm +launch projections. A real sealed bundle and validated production PowerPlan +are absent from this checkout, so offline preflight remains fail-closed. The +synthetic bundle is complete for evaluator tests but cannot enable a provider +run. A future caller must keep private evaluator material unmounted and +inaccessible to the runner identity; this protocol does not claim ACL or +process-isolation proof. diff --git a/benchmarks/corpus/router-cache/campaign.json b/benchmarks/corpus/router-cache/campaign.json index 176bee3..a48ca0a 100644 --- a/benchmarks/corpus/router-cache/campaign.json +++ b/benchmarks/corpus/router-cache/campaign.json @@ -1,9 +1,6 @@ { - "schema": "needle.multi-task-campaign/1", - "task_ids": [ - "ripgrep-glob-case-insensitive-locate-calibration", - "ripgrep-crlf-trace-calibration" - ], + "schema": "needle.multi-task-campaign/2", + "schedule_digest": "b3:c853b4ed84d8e9119013d788a7c769c63fd77f63a947706fbe4c128b65ed51d5", "paid_arms": [ "frontier_direct", "needle_miss" @@ -18,7 +15,6 @@ "native_subagent", "escalation" ], - "repetitions_per_task": 1, "automatic_retries": false, "statistical_claim": false, "bootstrap_resamples": 10000, diff --git a/benchmarks/corpus/router-cache/legacy-offline-campaign.json b/benchmarks/corpus/router-cache/legacy-offline-campaign.json new file mode 100644 index 0000000..765da08 --- /dev/null +++ b/benchmarks/corpus/router-cache/legacy-offline-campaign.json @@ -0,0 +1,26 @@ +{ + "schema": "needle.multi-task-campaign/1", + "task_ids": [ + "ripgrep-glob-case-insensitive-locate-calibration", + "ripgrep-crlf-trace-calibration" + ], + "paid_arms": ["frontier_direct", "needle_miss"], + "offline_cache_arms": ["exact_hit", "partial_hit", "irrelevant_mutation", "relevant_mutation"], + "deferred_diagnostic_arms": ["native_subagent", "escalation"], + "repetitions_per_task": 1, + "automatic_retries": false, + "statistical_claim": false, + "bootstrap_resamples": 10000, + "one_sided_alpha_basis_points": 500, + "target_power_basis_points": 9000, + "budget_reserve": { + "main_turn_microcredits": 2633875, + "extra_main_turns_per_needle_observation": 2, + "worker_microcredits": 2034130, + "extra_workers_per_needle_observation": 0, + "evidence": [ + "benchmarks/results/live/routing-and-cache-calibration.md", + "benchmarks/results/historical/cache-mutation-calibration.md" + ] + } +} diff --git a/benchmarks/corpus/router-cache/legacy-offline-manifest.json b/benchmarks/corpus/router-cache/legacy-offline-manifest.json new file mode 100644 index 0000000..d37608d --- /dev/null +++ b/benchmarks/corpus/router-cache/legacy-offline-manifest.json @@ -0,0 +1,70 @@ +{ + "schema": "needle.frozen-corpus/3", + "frozen_unix_ms": 1785412789576, + "arms": [ + "frontier_direct", + "native_subagent", + "needle_miss", + "exact_hit", + "partial_hit", + "escalation", + "irrelevant_mutation", + "relevant_mutation" + ], + "cost_model_path": "cost-model.json", + "cost_model_digest": "b3:a517364a8e5cdf1af49ea728c150c2bd2cff13503888175bf5da2471cfe8a74a", + "next_pilot_path": "minimal-live-pilot.json", + "next_pilot_digest": "b3:18a434227eb9abf2381eb0a65348f0dba66301ccba7dbdeb57c7933b078180cb", + "campaign_path": "legacy-offline-campaign.json", + "campaign_digest": "b3:883d16a6d66049e7a68fe3d9d70dbf7e46186bca5f9670eecc48e52f6abc0e26", + "tasks": [ + { + "id": "ripgrep-glob-case-insensitive-locate-calibration", + "route": "locate_implementation", + "split": "calibration", + "repository_url": "https://github.com/BurntSushi/ripgrep.git", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "prompt": "Where is ripgrep's --glob-case-insensitive option implemented, and which focused test demonstrates its behavior? Give the primary location and only the supporting evidence needed to continue.", + "oracle_path": "oracles/ripgrep-glob-case-insensitive-locate.json", + "oracle_digest": "b3:efc9f2b932008a42393bad9145d1db1dc420bade8778889d7fcf83d6c9734b7f", + "test_identifier": "misc::glob_always_case_insensitive", + "focused_command": ["cargo", "test", "--offline", "--test", "integration", "misc::glob_always_case_insensitive", "--", "--exact"] + }, + { + "id": "ripgrep-crlf-trace-calibration", + "route": "trace_state_flow", + "split": "calibration", + "repository_url": "https://github.com/BurntSushi/ripgrep.git", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "prompt": "Trace how ripgrep's --crlf option changes matching and search line terminators, and name a focused test that proves the default scenario.", + "oracle_path": "oracles/ripgrep-crlf-trace.json", + "oracle_digest": "b3:db68c75eebfab64593825a1563facc1e67634ad15e32dcccbd23c83d655b5059", + "test_identifier": "feature::f416_crlf", + "focused_command": ["cargo", "test", "--offline", "--test", "integration", "feature::f416_crlf", "--", "--exact"] + }, + { + "id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "repository_url": "https://github.com/BurntSushi/ripgrep.git", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "prompt": "Find the implementation that makes ripgrep's --no-ignore-vcs option stop applying Git ignore rules, and identify a focused test for that behavior.", + "oracle_path": "oracles/ripgrep-no-ignore-vcs-locate.json", + "oracle_digest": "b3:4eac2f4e630f8a457de9bcfa1cfdd3adef60fd46e399d24ac7479c2b1906398d", + "test_identifier": "feature::f68_no_ignore_vcs", + "focused_command": ["cargo", "test", "--offline", "--test", "integration", "feature::f68_no_ignore_vcs", "--", "--exact"] + }, + { + "id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "repository_url": "https://github.com/BurntSushi/ripgrep.git", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "prompt": "Trace how ripgrep's --null-data option changes matching and searching from newline-delimited input to NUL-delimited input, with a focused test for the behavior.", + "oracle_path": "oracles/ripgrep-null-data-trace.json", + "oracle_digest": "b3:e8efb461add472614a36eb920a0c0aa1857e13fe4834aa12cc2582d9e8df4661", + "test_identifier": "feature::f993_null_data", + "focused_command": ["cargo", "test", "--offline", "--test", "integration", "feature::f993_null_data", "--", "--exact"] + } + ] +} diff --git a/benchmarks/corpus/router-cache/manifest.json b/benchmarks/corpus/router-cache/manifest.json index 80e46e3..a92aecc 100644 --- a/benchmarks/corpus/router-cache/manifest.json +++ b/benchmarks/corpus/router-cache/manifest.json @@ -1,5 +1,5 @@ { - "schema": "needle.frozen-corpus/3", + "schema": "needle.frozen-corpus/4", "frozen_unix_ms": 1785412789576, "arms": [ "frontier_direct", @@ -16,7 +16,13 @@ "next_pilot_path": "minimal-live-pilot.json", "next_pilot_digest": "b3:18a434227eb9abf2381eb0a65348f0dba66301ccba7dbdeb57c7933b078180cb", "campaign_path": "campaign.json", - "campaign_digest": "b3:207fb6ef4a29606c76054b3dcd27641cb10ae114c35501b7b798fb46e7c1a597", + "campaign_digest": "b3:082239af1ebe785555801af74c817c5e49c143759cd3e62ef6607f54dadf5e1d", + "schedule_path": "schedule.json", + "schedule_digest": "b3:c853b4ed84d8e9119013d788a7c769c63fd77f63a947706fbe4c128b65ed51d5", + "power_plan_path": "power-plan.json", + "power_plan_digest": "b3:8605f040792744907aa94d3ce017a211b774c16521dbe744acb7a1b587647630", + "sealed_bundle_schema": "needle.sealed-oracle-index/1", + "sealed_bundle_digest": "b3:3750e933fa1226fed1892dc4ba3ddad4f84fbf6bd58135d8f26355103dc681c7", "tasks": [ { "id": "ripgrep-glob-case-insensitive-locate-calibration", @@ -24,20 +30,14 @@ "split": "calibration", "repository_url": "https://github.com/BurntSushi/ripgrep.git", "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", - "prompt": "Where is ripgrep's --glob-case-insensitive option implemented, and which focused test demonstrates its behavior? Give the primary location and only the supporting evidence needed to continue.", - "oracle_path": "oracles/ripgrep-glob-case-insensitive-locate.json", - "oracle_digest": "b3:efc9f2b932008a42393bad9145d1db1dc420bade8778889d7fcf83d6c9734b7f", - "test_identifier": "misc::glob_always_case_insensitive", - "focused_command": [ - "cargo", - "test", - "--offline", - "--test", - "integration", - "misc::glob_always_case_insensitive", - "--", - "--exact" - ] + "prompt": "Where is ripgrep's glob-case-insensitive option implemented, and which focused test demonstrates its behavior? Give the primary location and only the supporting evidence needed to continue.", + "material_class": "synthetic", + "focused_test_policy": { + "identity": "ripgrep.synthetic.glob-case-insensitive.v1", + "commitment": "b3:a7d0811cef052068f9640398720e65b19e7f1a9e0990facd03e42a9e97e26f4f" + }, + "oracle_schema": "needle.sealed-oracle/1", + "oracle_digest": "b3:7a6787e9e7aaa0bcc147e69a7dd32b226390dabc2e854ed7afa2f1d7b36c49b6" }, { "id": "ripgrep-crlf-trace-calibration", @@ -45,20 +45,14 @@ "split": "calibration", "repository_url": "https://github.com/BurntSushi/ripgrep.git", "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", - "prompt": "Trace how ripgrep's --crlf option changes matching and search line terminators, and name a focused test that proves the default scenario.", - "oracle_path": "oracles/ripgrep-crlf-trace.json", - "oracle_digest": "b3:db68c75eebfab64593825a1563facc1e67634ad15e32dcccbd23c83d655b5059", - "test_identifier": "feature::f416_crlf", - "focused_command": [ - "cargo", - "test", - "--offline", - "--test", - "integration", - "feature::f416_crlf", - "--", - "--exact" - ] + "prompt": "Trace how ripgrep's crlf option changes matching and search line terminators, and name a focused test that proves the default scenario.", + "material_class": "synthetic", + "focused_test_policy": { + "identity": "ripgrep.synthetic.crlf.v1", + "commitment": "b3:27d2e8e1cecfba3b600bcb411ffea48547319aa3fefd6704c46dd7c13fa5c250" + }, + "oracle_schema": "needle.sealed-oracle/1", + "oracle_digest": "b3:4e4053553759f507b41bd3b3834691f53b2b743da9b0aa866c6f53764044213d" }, { "id": "ripgrep-no-ignore-vcs-locate-holdout", @@ -66,20 +60,14 @@ "split": "holdout", "repository_url": "https://github.com/BurntSushi/ripgrep.git", "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", - "prompt": "Find the implementation that makes ripgrep's --no-ignore-vcs option stop applying Git ignore rules, and identify a focused test for that behavior.", - "oracle_path": "oracles/ripgrep-no-ignore-vcs-locate.json", - "oracle_digest": "b3:4eac2f4e630f8a457de9bcfa1cfdd3adef60fd46e399d24ac7479c2b1906398d", - "test_identifier": "feature::f68_no_ignore_vcs", - "focused_command": [ - "cargo", - "test", - "--offline", - "--test", - "integration", - "feature::f68_no_ignore_vcs", - "--", - "--exact" - ] + "prompt": "Find the implementation that makes ripgrep's no-ignore-vcs option stop applying repository ignore rules, and identify a focused test for that behavior.", + "material_class": "synthetic", + "focused_test_policy": { + "identity": "ripgrep.synthetic.no-ignore-vcs.v1", + "commitment": "b3:ff0c45be0d9ea0e25a4c7a4f6dcac9b153a05b08c7f23d723011e5bf4e58de57" + }, + "oracle_schema": "needle.sealed-oracle/1", + "oracle_digest": "b3:96c6568343f73fb983c8cae8a7f483d339e14f42d5e6af0c36b2dfdc942f9571" }, { "id": "ripgrep-null-data-trace-holdout", @@ -87,20 +75,14 @@ "split": "holdout", "repository_url": "https://github.com/BurntSushi/ripgrep.git", "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", - "prompt": "Trace how ripgrep's --null-data option changes matching and searching from newline-delimited input to NUL-delimited input, with a focused test for the behavior.", - "oracle_path": "oracles/ripgrep-null-data-trace.json", - "oracle_digest": "b3:e8efb461add472614a36eb920a0c0aa1857e13fe4834aa12cc2582d9e8df4661", - "test_identifier": "feature::f993_null_data", - "focused_command": [ - "cargo", - "test", - "--offline", - "--test", - "integration", - "feature::f993_null_data", - "--", - "--exact" - ] + "prompt": "Trace how ripgrep's null-data option changes matching and searching from newline-delimited input to NUL-delimited input, with a focused test for the behavior.", + "material_class": "synthetic", + "focused_test_policy": { + "identity": "ripgrep.synthetic.null-data.v1", + "commitment": "b3:2cc53eb4760e8106d5638ec54c545d7447fda90a10a35668e7707ae5cfe1b21e" + }, + "oracle_schema": "needle.sealed-oracle/1", + "oracle_digest": "b3:6900cdf524dfaab61c434864f9eaede519737cc9fd30d7b418ef5003d5042beb" } ] } diff --git a/benchmarks/corpus/router-cache/oracles/README.md b/benchmarks/corpus/router-cache/oracles/README.md new file mode 100644 index 0000000..94bda9d --- /dev/null +++ b/benchmarks/corpus/router-cache/oracles/README.md @@ -0,0 +1,12 @@ +# Synthetic evaluator fixtures + +These answer-bearing JSON documents are retained solely for deterministic +legacy/evaluator tests. They are not referenced by the public +`needle.frozen-corpus/4` manifest, are not sealed production material, and +must never be used as independent provider evidence. + +The checked-in `../synthetic-sealed/` index and four documents exercise the +sealed evaluator contract and exact byte commitments. They are synthetic and +remain provider-ineligible. Private production bundles must stay unmounted and +inaccessible to the runner identity; this fixture does not claim ACL or +process-isolation proof. diff --git a/benchmarks/corpus/router-cache/power-plan.json b/benchmarks/corpus/router-cache/power-plan.json new file mode 100644 index 0000000..d7367fc --- /dev/null +++ b/benchmarks/corpus/router-cache/power-plan.json @@ -0,0 +1,32 @@ +{ + "schema": "needle.power-plan/1", + "plan_id": "synthetic-router-cache-v1", + "manifest_digest": "b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a", + "campaign_digest": "b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864", + "calibration_input_digest": "b3:4d27a8427bd3655d59e88c37efadcccfca4a7d82cd4a4324708180aea31289c0", + "estimator_revision": "issue-7-structural-v1", + "alpha_basis_points": 500, + "target_power_basis_points": 9000, + "routes": [ + { + "route": "locate_implementation", + "baseline_arm": "frontier_direct", + "treatment_arm": "needle_miss", + "pair_key": "task_id:repetition", + "observed_log_ratio_mean": -0.4, + "observed_log_ratio_stddev": 0.2, + "required_pairs": 1 + }, + { + "route": "trace_state_flow", + "baseline_arm": "frontier_direct", + "treatment_arm": "needle_miss", + "pair_key": "task_id:repetition", + "observed_log_ratio_mean": -0.4, + "observed_log_ratio_stddev": 0.2, + "required_pairs": 1 + } + ], + "validated": true, + "synthetic": true +} diff --git a/benchmarks/corpus/router-cache/schedule.json b/benchmarks/corpus/router-cache/schedule.json new file mode 100644 index 0000000..a187567 --- /dev/null +++ b/benchmarks/corpus/router-cache/schedule.json @@ -0,0 +1,28 @@ +{ + "schema": "needle.corpus-schedule/1", + "manifest_digest": "b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a", + "power_plan_digest": "b3:8605f040792744907aa94d3ce017a211b774c16521dbe744acb7a1b587647630", + "automatic_retries": false, + "entries": [ + {"task_id":"ripgrep-glob-case-insensitive-locate-calibration","route":"locate_implementation","split":"calibration","arm":"frontier_direct","repetition":0}, + {"task_id":"ripgrep-glob-case-insensitive-locate-calibration","route":"locate_implementation","split":"calibration","arm":"needle_miss","repetition":0}, + {"task_id":"ripgrep-crlf-trace-calibration","route":"trace_state_flow","split":"calibration","arm":"frontier_direct","repetition":0}, + {"task_id":"ripgrep-crlf-trace-calibration","route":"trace_state_flow","split":"calibration","arm":"needle_miss","repetition":0}, + {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"frontier_direct","repetition":0}, + {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"native_subagent","repetition":0}, + {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"needle_miss","repetition":0}, + {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"exact_hit","repetition":0}, + {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"partial_hit","repetition":0}, + {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"escalation","repetition":0}, + {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"irrelevant_mutation","repetition":0}, + {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"relevant_mutation","repetition":0}, + {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"frontier_direct","repetition":0}, + {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"native_subagent","repetition":0}, + {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"needle_miss","repetition":0}, + {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"exact_hit","repetition":0}, + {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"partial_hit","repetition":0}, + {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"escalation","repetition":0}, + {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"irrelevant_mutation","repetition":0}, + {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"relevant_mutation","repetition":0} + ] +} diff --git a/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-crlf-trace-calibration.json b/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-crlf-trace-calibration.json new file mode 100644 index 0000000..5a5710c --- /dev/null +++ b/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-crlf-trace-calibration.json @@ -0,0 +1,16 @@ +{ + "schema": "needle.sealed-oracle/1", + "task_id": "ripgrep-crlf-trace-calibration", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "focused_test_policy": {"identity": "ripgrep.synthetic.crlf.v1", "commitment": "b3:27d2e8e1cecfba3b600bcb411ffea48547319aa3fefd6704c46dd7c13fa5c250"}, + "focused_test": {"identifier": "feature::f416_crlf", "argv": ["cargo", "test", "--offline", "--test", "integration", "feature::f416_crlf", "--", "--exact"]}, + "quality": { + "required_files": ["crates/core/flags/defs.rs", "crates/core/flags/hiargs.rs"], + "required_symbols": [], + "required_claims": ["crlf(true)", "LineTerminator::crlf"], + "forbidden_claims": [], + "focused_test_command": "feature::f416_crlf", + "accepted_focused_test_identifiers": ["feature::f416_crlf", "f416_crlf"], + "focused_test_required": true + } +} diff --git a/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-glob-case-insensitive-locate-calibration.json b/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-glob-case-insensitive-locate-calibration.json new file mode 100644 index 0000000..7836324 --- /dev/null +++ b/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-glob-case-insensitive-locate-calibration.json @@ -0,0 +1,16 @@ +{ + "schema": "needle.sealed-oracle/1", + "task_id": "ripgrep-glob-case-insensitive-locate-calibration", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "focused_test_policy": {"identity": "ripgrep.synthetic.glob-case-insensitive.v1", "commitment": "b3:a7d0811cef052068f9640398720e65b19e7f1a9e0990facd03e42a9e97e26f4f"}, + "focused_test": {"identifier": "misc::glob_always_case_insensitive", "argv": ["cargo", "test", "--offline", "--test", "integration", "misc::glob_always_case_insensitive", "--", "--exact"]}, + "quality": { + "required_files": ["crates/core/flags/hiargs.rs"], + "required_symbols": ["globs"], + "required_claims": [], + "forbidden_claims": [], + "focused_test_command": "misc::glob_always_case_insensitive", + "accepted_focused_test_identifiers": ["misc::glob_always_case_insensitive"], + "focused_test_required": true + } +} diff --git a/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-no-ignore-vcs-locate-holdout.json b/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-no-ignore-vcs-locate-holdout.json new file mode 100644 index 0000000..ba07d71 --- /dev/null +++ b/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-no-ignore-vcs-locate-holdout.json @@ -0,0 +1,16 @@ +{ + "schema": "needle.sealed-oracle/1", + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "focused_test_policy": {"identity": "ripgrep.synthetic.no-ignore-vcs.v1", "commitment": "b3:ff0c45be0d9ea0e25a4c7a4f6dcac9b153a05b08c7f23d723011e5bf4e58de57"}, + "focused_test": {"identifier": "feature::f68_no_ignore_vcs", "argv": ["cargo", "test", "--offline", "--test", "integration", "feature::f68_no_ignore_vcs", "--", "--exact"]}, + "quality": { + "required_files": ["crates/core/flags/hiargs.rs"], + "required_symbols": ["no_ignore_vcs"], + "required_claims": [], + "forbidden_claims": [], + "focused_test_command": "feature::f68_no_ignore_vcs", + "accepted_focused_test_identifiers": ["feature::f68_no_ignore_vcs", "f68_no_ignore_vcs"], + "focused_test_required": true + } +} diff --git a/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-null-data-trace-holdout.json b/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-null-data-trace-holdout.json new file mode 100644 index 0000000..d9fecf8 --- /dev/null +++ b/benchmarks/corpus/router-cache/synthetic-sealed/docs/ripgrep-null-data-trace-holdout.json @@ -0,0 +1,16 @@ +{ + "schema": "needle.sealed-oracle/1", + "task_id": "ripgrep-null-data-trace-holdout", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "focused_test_policy": {"identity": "ripgrep.synthetic.null-data.v1", "commitment": "b3:2cc53eb4760e8106d5638ec54c545d7447fda90a10a35668e7707ae5cfe1b21e"}, + "focused_test": {"identifier": "feature::f993_null_data", "argv": ["cargo", "test", "--offline", "--test", "integration", "feature::f993_null_data", "--", "--exact"]}, + "quality": { + "required_files": ["crates/core/flags/defs.rs", "crates/core/flags/hiargs.rs"], + "required_symbols": [], + "required_claims": ["null_data", "NUL"], + "forbidden_claims": [], + "focused_test_command": "feature::f993_null_data", + "accepted_focused_test_identifiers": ["feature::f993_null_data", "f993_null_data"], + "focused_test_required": true + } +} diff --git a/benchmarks/corpus/router-cache/synthetic-sealed/index.json b/benchmarks/corpus/router-cache/synthetic-sealed/index.json new file mode 100644 index 0000000..dcb55f8 --- /dev/null +++ b/benchmarks/corpus/router-cache/synthetic-sealed/index.json @@ -0,0 +1,41 @@ +{ + "schema": "needle.sealed-oracle-index/1", + "entries": [ + { + "task_id": "ripgrep-glob-case-insensitive-locate-calibration", + "material_class": "synthetic", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "oracle_schema": "needle.sealed-oracle/1", + "oracle_digest": "b3:7a6787e9e7aaa0bcc147e69a7dd32b226390dabc2e854ed7afa2f1d7b36c49b6", + "focused_test_policy": {"identity": "ripgrep.synthetic.glob-case-insensitive.v1", "commitment": "b3:a7d0811cef052068f9640398720e65b19e7f1a9e0990facd03e42a9e97e26f4f"}, + "relative_path": "docs/ripgrep-glob-case-insensitive-locate-calibration.json" + }, + { + "task_id": "ripgrep-crlf-trace-calibration", + "material_class": "synthetic", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "oracle_schema": "needle.sealed-oracle/1", + "oracle_digest": "b3:4e4053553759f507b41bd3b3834691f53b2b743da9b0aa866c6f53764044213d", + "focused_test_policy": {"identity": "ripgrep.synthetic.crlf.v1", "commitment": "b3:27d2e8e1cecfba3b600bcb411ffea48547319aa3fefd6704c46dd7c13fa5c250"}, + "relative_path": "docs/ripgrep-crlf-trace-calibration.json" + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "material_class": "synthetic", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "oracle_schema": "needle.sealed-oracle/1", + "oracle_digest": "b3:96c6568343f73fb983c8cae8a7f483d339e14f42d5e6af0c36b2dfdc942f9571", + "focused_test_policy": {"identity": "ripgrep.synthetic.no-ignore-vcs.v1", "commitment": "b3:ff0c45be0d9ea0e25a4c7a4f6dcac9b153a05b08c7f23d723011e5bf4e58de57"}, + "relative_path": "docs/ripgrep-no-ignore-vcs-locate-holdout.json" + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "material_class": "synthetic", + "repository_sha": "4649aa9700619f94cf9c66876e9549d83420e16c", + "oracle_schema": "needle.sealed-oracle/1", + "oracle_digest": "b3:6900cdf524dfaab61c434864f9eaede519737cc9fd30d7b418ef5003d5042beb", + "focused_test_policy": {"identity": "ripgrep.synthetic.null-data.v1", "commitment": "b3:2cc53eb4760e8106d5638ec54c545d7447fda90a10a35668e7707ae5cfe1b21e"}, + "relative_path": "docs/ripgrep-null-data-trace-holdout.json" + } + ] +} diff --git a/crates/needle-app/src/main.rs b/crates/needle-app/src/main.rs index 6b99125..cb4c217 100644 --- a/crates/needle-app/src/main.rs +++ b/crates/needle-app/src/main.rs @@ -781,13 +781,13 @@ fn mcp_contract_microbench_run(arguments: &[String]) -> Result<(), AppError> { } fn quality_oracle_replay(arguments: &[String]) -> Result<(), AppError> { - let manifest = option_value(arguments, "--manifest") - .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from(minimal_live_pilot::protocol::DEFAULT_MANIFEST)); + let manifest = option_value(arguments, "--manifest").map(PathBuf::from).unwrap_or_else(|| { + PathBuf::from(minimal_live_pilot::protocol::DEFAULT_LEGACY_OFFLINE_MANIFEST) + }); let task_id = required_value(arguments, "--task-id")?; let response_path = PathBuf::from(required_value(arguments, "--response")?); let response = fs::read_to_string(&response_path)?; - let protocol = minimal_live_pilot::protocol::load_protocol(&manifest)?; + let protocol = minimal_live_pilot::protocol::load_legacy_offline_protocol(&manifest)?; let (task, oracle) = protocol.campaign_task(&task_id)?; let spec = minimal_live_pilot::protocol::quality_spec_for_task(task, oracle)?; let quality = needle_bench::QualityOracleResult::evaluate(&spec, &response, None); diff --git a/crates/needle-app/src/minimal_live_pilot.rs b/crates/needle-app/src/minimal_live_pilot.rs index a99ba8e..7d8b3db 100644 --- a/crates/needle-app/src/minimal_live_pilot.rs +++ b/crates/needle-app/src/minimal_live_pilot.rs @@ -34,11 +34,11 @@ mod supervised_main; use direct_main::{DirectObservation, observe as observe_direct_main}; use protocol::{ - DEFAULT_MANIFEST, DEFAULT_PRICING, MULTI_NEED_EXTRA_MAIN_TURN_RESERVES, - MULTI_NEED_EXTRA_WORKER_RESERVES, MULTI_NEED_MAIN_TURN_RESERVE_MICROCREDITS, - MULTI_NEED_WORKER_RESERVE_MICROCREDITS, Protocol, TRACE_REUSE_TASK_ID, - coverage_hit_quality_spec, load_pricing, load_protocol, quality_spec, test_plan, - validate_source, workspace_path, + DEFAULT_LEGACY_OFFLINE_MANIFEST, DEFAULT_MANIFEST, DEFAULT_PRICING, + MULTI_NEED_EXTRA_MAIN_TURN_RESERVES, MULTI_NEED_EXTRA_WORKER_RESERVES, + MULTI_NEED_MAIN_TURN_RESERVE_MICROCREDITS, MULTI_NEED_WORKER_RESERVE_MICROCREDITS, Protocol, + TRACE_REUSE_TASK_ID, coverage_hit_quality_spec, load_legacy_offline_protocol, load_pricing, + load_protocol, quality_spec, test_plan, validate_source, workspace_path, }; use supervised_main::SupervisedMain; @@ -341,13 +341,22 @@ pub(super) fn run(arguments: &[String]) -> Result<(), AppError> { )); } - let manifest_path = option_value(arguments, "--corpus") - .map(PathBuf::from) - .unwrap_or_else(|| workspace_path(DEFAULT_MANIFEST)); + let manifest_path = + option_value(arguments, "--corpus").map(PathBuf::from).unwrap_or_else(|| { + workspace_path(if execute_offline { + DEFAULT_LEGACY_OFFLINE_MANIFEST + } else { + DEFAULT_MANIFEST + }) + }); let pricing_path = option_value(arguments, "--pricing-snapshot") .map(PathBuf::from) .unwrap_or_else(|| workspace_path(DEFAULT_PRICING)); - let mut protocol = load_protocol(&manifest_path)?; + let mut protocol = if execute_offline { + load_legacy_offline_protocol(&manifest_path)? + } else { + load_protocol(&manifest_path)? + }; if trace_reuse { protocol.select_campaign_task(TRACE_REUSE_TASK_ID)?; } @@ -1716,6 +1725,7 @@ fn now_ms() -> u64 { #[cfg(test)] mod tests { + use super::protocol::DEFAULT_LEGACY_OFFLINE_MANIFEST; use super::*; use needle_bench::ProcessExecutionStatus; @@ -1876,7 +1886,8 @@ mod tests { #[test] fn economic_stage_budget_adds_main_only_and_keeps_reserves_explicit() { - let protocol = load_protocol(&workspace_path(DEFAULT_MANIFEST)).unwrap(); + let protocol = + load_legacy_offline_protocol(&workspace_path(DEFAULT_LEGACY_OFFLINE_MANIFEST)).unwrap(); let budget = protocol.economic_stage_budget().unwrap(); assert_eq!(budget.main_only_microcredits, 11_158_850); assert_eq!(budget.needle_miss_microcredits, 5_311_720); diff --git a/crates/needle-app/src/minimal_live_pilot/protocol.rs b/crates/needle-app/src/minimal_live_pilot/protocol.rs index ca19154..126da13 100644 --- a/crates/needle-app/src/minimal_live_pilot/protocol.rs +++ b/crates/needle-app/src/minimal_live_pilot/protocol.rs @@ -13,6 +13,8 @@ use std::path::{Path, PathBuf}; use std::process::Command; pub(crate) const DEFAULT_MANIFEST: &str = "benchmarks/corpus/router-cache/manifest.json"; +pub(crate) const DEFAULT_LEGACY_OFFLINE_MANIFEST: &str = + "benchmarks/corpus/router-cache/legacy-offline-manifest.json"; pub(crate) const DEFAULT_PRICING: &str = "fixtures/openai-codex-pricing-2026-07-27.json"; const PILOT_TASK_ID: &str = "ripgrep-glob-case-insensitive-locate-calibration"; pub(crate) const TRACE_REUSE_TASK_ID: &str = "ripgrep-crlf-trace-calibration"; @@ -210,12 +212,38 @@ fn arm_estimate(model: &CampaignCostModel, arm: FinalArm) -> Result Result { + let manifest_path = canonical_child_path(manifest_path)?; + let manifest: FrozenCorpusManifest = serde_json::from_slice(&fs::read(&manifest_path)?)?; + if manifest.schema != "needle.frozen-corpus/4" { + return Err(AppError::Experiment( + "legacy frozen corpus manifests are restricted to the explicit offline replay loader" + .to_owned(), + )); + } + Err(AppError::Experiment( + "frozen corpus v4 requires an evaluator-owned production sealed bundle; provider execution is fail-closed in this runner" + .to_owned(), + )) +} + +/// Explicit legacy offline loader used only by deterministic replay/simulator +/// paths. Keeping this separate from `load_protocol` prevents a caller- +/// supplied v2/v3 manifest from reaching a provider/live command. +pub(crate) fn load_legacy_offline_protocol(manifest_path: &Path) -> Result { let manifest_path = canonical_child_path(manifest_path)?; let directory = manifest_path .parent() .ok_or_else(|| AppError::Experiment("frozen corpus has no parent directory".to_owned()))?; let manifest: FrozenCorpusManifest = serde_json::from_slice(&fs::read(&manifest_path)?)?; + if !matches!(manifest.schema.as_str(), "needle.frozen-corpus/2" | "needle.frozen-corpus/3") { + return Err(AppError::Experiment( + "legacy offline loader accepts only frozen corpus v2/v3 manifests".to_owned(), + )); + } let errors = validate_frozen_manifest(&manifest); if !errors.is_empty() { return Err(AppError::Experiment(format!( @@ -567,7 +595,8 @@ mod tests { #[test] fn frozen_protocol_is_exactly_one_miss_then_one_hit() { - let protocol = load_protocol(&workspace_path(DEFAULT_MANIFEST)).unwrap(); + let protocol = + load_legacy_offline_protocol(&workspace_path(DEFAULT_LEGACY_OFFLINE_MANIFEST)).unwrap(); assert_eq!(protocol.pilot.paid_arms, [FinalArm::NeedleMiss, FinalArm::ExactHit]); assert_eq!(protocol.estimated_budget_microcredits, 10_936_895); assert_eq!(protocol.task().id, PILOT_TASK_ID); @@ -575,7 +604,8 @@ mod tests { #[test] fn current_multi_task_budget_contains_only_the_two_paid_arms_and_reserves() { - let protocol = load_protocol(&workspace_path(DEFAULT_MANIFEST)).unwrap(); + let protocol = + load_legacy_offline_protocol(&workspace_path(DEFAULT_LEGACY_OFFLINE_MANIFEST)).unwrap(); let budget = protocol.multi_task_stage_budget().unwrap(); assert_eq!(protocol.campaign.task_ids.len(), 2); assert_eq!(budget.task_count, 2); @@ -587,7 +617,8 @@ mod tests { #[test] fn natural_r43_answer_satisfies_the_public_quality_gate() { - let protocol = load_protocol(&workspace_path(DEFAULT_MANIFEST)).unwrap(); + let protocol = + load_legacy_offline_protocol(&workspace_path(DEFAULT_LEGACY_OFFLINE_MANIFEST)).unwrap(); let spec = quality_spec(&protocol).unwrap(); let response = "The primary implementation is `globs` in \ `crates/core/flags/hiargs.rs` (lines 1209-1219). The focused test is \ @@ -598,7 +629,8 @@ mod tests { #[test] fn coverage_hit_oracle_does_not_require_an_unrequested_test() { - let protocol = load_protocol(&workspace_path(DEFAULT_MANIFEST)).unwrap(); + let protocol = + load_legacy_offline_protocol(&workspace_path(DEFAULT_LEGACY_OFFLINE_MANIFEST)).unwrap(); let spec = coverage_hit_quality_spec(&protocol).unwrap(); let response = "The primary implementation is in `crates/core/flags/hiargs.rs`."; let result = needle_bench::QualityOracleResult::evaluate(&spec, response, None); @@ -608,7 +640,8 @@ mod tests { #[test] fn trace_campaign_oracle_accepts_declared_semantic_evidence_and_test_alternative() { - let protocol = load_protocol(&workspace_path(DEFAULT_MANIFEST)).unwrap(); + let protocol = + load_legacy_offline_protocol(&workspace_path(DEFAULT_LEGACY_OFFLINE_MANIFEST)).unwrap(); let (task, oracle) = protocol.campaign_task("ripgrep-crlf-trace-calibration").unwrap(); let spec = quality_spec_for_task(task, oracle).unwrap(); let response = "`--crlf` is parsed in crates/core/flags/defs.rs. In \ @@ -620,7 +653,8 @@ mod tests { #[test] fn trace_campaign_oracle_rejects_an_incomplete_runtime_explanation() { - let protocol = load_protocol(&workspace_path(DEFAULT_MANIFEST)).unwrap(); + let protocol = + load_legacy_offline_protocol(&workspace_path(DEFAULT_LEGACY_OFFLINE_MANIFEST)).unwrap(); let (task, oracle) = protocol.campaign_task("ripgrep-crlf-trace-calibration").unwrap(); let spec = quality_spec_for_task(task, oracle).unwrap(); let response = "`--crlf` appears in crates/core/flags/defs.rs and \ @@ -629,4 +663,10 @@ mod tests { assert!(!result.passed); assert_eq!(result.failures, vec!["required_claims"]); } + + #[test] + fn provider_loader_rejects_the_public_v4_manifest() { + let error = load_protocol(&workspace_path(DEFAULT_MANIFEST)).unwrap_err(); + assert!(error.to_string().contains("evaluator-owned production sealed bundle")); + } } diff --git a/crates/needle-app/src/worker_live_diagnostic.rs b/crates/needle-app/src/worker_live_diagnostic.rs index 6e9610f..3402996 100644 --- a/crates/needle-app/src/worker_live_diagnostic.rs +++ b/crates/needle-app/src/worker_live_diagnostic.rs @@ -732,13 +732,23 @@ impl Drop for TemporaryRunRoot { #[cfg(test)] mod tests { use super::*; + use crate::minimal_live_pilot::protocol::{ + DEFAULT_LEGACY_OFFLINE_MANIFEST, load_legacy_offline_protocol, + }; #[test] fn worker_budget_matches_the_accepted_historical_component() { - let protocol = load_protocol(&workspace_path(DEFAULT_MANIFEST)).unwrap(); + let protocol = + load_legacy_offline_protocol(&workspace_path(DEFAULT_LEGACY_OFFLINE_MANIFEST)).unwrap(); assert_eq!(worker_budget(&protocol).unwrap(), OBSERVED_WORKER_BUDGET_MICROCREDITS); } + #[test] + fn provider_worker_loader_rejects_the_public_v4_manifest() { + let error = load_protocol(&workspace_path(DEFAULT_MANIFEST)).unwrap_err(); + assert!(error.to_string().contains("evaluator-owned production sealed bundle")); + } + #[test] fn frozen_marker_is_unversioned_and_semantic() { let need = NeedIr::parse(WORKER_MARKER).unwrap().unwrap(); diff --git a/crates/needle-bench/src/bin/corpus-preflight.rs b/crates/needle-bench/src/bin/corpus-preflight.rs index aa40c10..b80a9c8 100644 --- a/crates/needle-bench/src/bin/corpus-preflight.rs +++ b/crates/needle-bench/src/bin/corpus-preflight.rs @@ -1,4 +1,7 @@ -use needle_bench::{FrozenCorpusManifest, preflight_frozen_corpus}; +use needle_bench::{ + CorpusPreflightOptions, FrozenCorpusManifest, MAX_CORPUS_MANIFEST_BYTES, + preflight_exit_is_failure, preflight_frozen_corpus_with_options, read_bounded_file, +}; use std::env; use std::fs; use std::path::{Path, PathBuf}; @@ -17,7 +20,10 @@ fn run() -> Result<(), Box> { .canonicalize()?; if let Some(path) = option_path(&arguments, "--digest") { let path = absolute_from(&workspace, path); - println!("b3:{}", blake3::hash(&fs::read(path)?).to_hex()); + println!( + "b3:{}", + blake3::hash(&read_bounded_file(&path, MAX_CORPUS_MANIFEST_BYTES)?).to_hex() + ); return Ok(()); } let manifest_path = option_path(&arguments, "--manifest") @@ -31,24 +37,36 @@ fn run() -> Result<(), Box> { .unwrap_or_else(|| workspace.join("target/corpus-preflight/report.json")); let execute_focused_tests = arguments.iter().any(|argument| argument == "--execute-focused-tests"); - let manifest: FrozenCorpusManifest = serde_json::from_slice(&fs::read(&manifest_path)?)?; + let options = CorpusPreflightOptions { + schedule_path: option_path(&arguments, "--schedule") + .map(|path| absolute_from(&workspace, path)), + power_plan_path: option_path(&arguments, "--power-plan") + .map(|path| absolute_from(&workspace, path)), + sealed_bundle_index_path: option_path(&arguments, "--sealed-bundle-index") + .map(|path| absolute_from(&workspace, path)), + sealed_bundle_root: option_path(&arguments, "--sealed-bundle-root") + .map(|path| absolute_from(&workspace, path)), + }; + let manifest_bytes = read_bounded_file(&manifest_path, MAX_CORPUS_MANIFEST_BYTES)?; + let manifest: FrozenCorpusManifest = serde_json::from_slice(&manifest_bytes)?; let manifest_directory = manifest_path.parent().ok_or("manifest path must have a parent directory")?; - let report = preflight_frozen_corpus( + let report = preflight_frozen_corpus_with_options( &manifest, manifest_directory, &source_repository, execute_focused_tests, + &options, ); if let Some(parent) = output.parent() { fs::create_dir_all(parent)?; } fs::write(&output, serde_json::to_vec_pretty(&report)?)?; println!("{}", output.display()); - if !report.errors.is_empty() - || (execute_focused_tests - && report.tasks.iter().any(|task| task.focused_test_passed != Some(true))) - { + // Preflight is an offline diagnostic. Missing production-only material + // is reported in bounded fields and intentionally does not turn the + // command into a provider attempt or a shell-level failure. + if preflight_exit_is_failure(&report, &options, execute_focused_tests) { return Err("corpus preflight failed closed".into()); } Ok(()) diff --git a/crates/needle-bench/src/corpus.rs b/crates/needle-bench/src/corpus.rs index df96e04..8e11bff 100644 --- a/crates/needle-bench/src/corpus.rs +++ b/crates/needle-bench/src/corpus.rs @@ -1,4 +1,8 @@ -use crate::{FinalArm, FrozenCorpusManifest, validate_frozen_manifest}; +use crate::{ + CorpusMaterialClass, CorpusSchedule, FinalArm, FrozenCorpusManifest, PowerPlan, + SealedBundleValidationReport, build_launch_plan, read_bounded_file, validate_frozen_manifest, + validate_sealed_bundle, +}; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; use std::fs; @@ -39,6 +43,9 @@ pub struct CorpusTaskPreflight { pub focused_test_executed: bool, pub focused_test_passed: Option, pub errors: Vec, + pub material_class: CorpusMaterialClass, + pub focused_test_policy_valid: bool, + pub sealed_bundle_bound: bool, } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -102,10 +109,16 @@ pub struct PromotionBootstrap { #[serde(deny_unknown_fields)] pub struct MultiTaskCampaign { pub schema: String, + #[serde(default)] + pub schedule_digest: String, + /// v1-only calibration authority retained for legacy offline replay. + #[serde(default)] pub task_ids: Vec, pub paid_arms: Vec, pub offline_cache_arms: Vec, pub deferred_diagnostic_arms: Vec, + /// v1-only repetition authority retained for legacy offline replay. + #[serde(default)] pub repetitions_per_task: u32, pub automatic_retries: bool, pub statistical_claim: bool, @@ -115,6 +128,16 @@ pub struct MultiTaskCampaign { pub budget_reserve: CampaignBudgetReserve, } +/// Campaign commitment used by v2 PowerPlans. The schedule digest is the +/// forward reference from campaign to schedule, so it is deliberately blanked +/// for this non-circular semantic commitment; the manifest still commits the +/// campaign's exact raw bytes. +pub fn campaign_commitment(campaign: &MultiTaskCampaign) -> String { + let mut commitment = campaign.clone(); + commitment.schedule_digest.clear(); + format!("b3:{}", blake3::hash(&serde_json::to_vec(&commitment).unwrap_or_default()).to_hex()) +} + #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct CampaignBudgetReserve { @@ -139,12 +162,68 @@ pub struct CorpusPreflightReport { pub campaign_digest: Option, pub tasks: Vec, pub cost_readiness: CampaignCostReadiness, + pub provider_inputs_ready: bool, pub provider_run_ready: bool, + pub provider_blockers: Vec, pub errors: Vec, + pub manifest_errors: Vec, + pub schedule_valid: bool, + pub schedule_digest: Option, + pub schedule_errors: Vec, + pub power_plan_valid: bool, + pub power_plan_digest: Option, + pub power_plan_errors: Vec, + pub bundle_valid: bool, + pub bundle_digest: Option, + pub bundle_errors: Vec, + pub source_errors: Vec, +} + +#[derive(Clone, Debug, Default)] +pub struct CorpusPreflightOptions { + pub schedule_path: Option, + pub power_plan_path: Option, + pub sealed_bundle_index_path: Option, + pub sealed_bundle_root: Option, +} + +/// Shell-level predicate for the diagnostic CLI. Expected non-readiness of +/// the public synthetic protocol (missing private bundle or source checkout) +/// is exit-success; malformed protocol artifacts and explicitly supplied +/// invalid private material are exit-failure. +pub fn preflight_exit_is_failure( + report: &CorpusPreflightReport, + options: &CorpusPreflightOptions, + execute_focused_tests: bool, +) -> bool { + if !report.manifest_valid + || !report.schedule_valid + || !report.power_plan_valid + || (report.campaign_digest.is_some() && !report.campaign_valid) + || report.cost_readiness.full_protocol_calibration_budget_estimate_microcredits.is_none() + || (report.campaign_digest.is_some() + && report.cost_readiness.full_protocol_campaign_budget_estimate_microcredits.is_none()) + { + return true; + } + if (options.sealed_bundle_index_path.is_some() || options.sealed_bundle_root.is_some()) + && !report.bundle_valid + { + return true; + } + execute_focused_tests && report.tasks.iter().any(|task| task.focused_test_passed != Some(true)) } pub fn corpus_digest(manifest: &FrozenCorpusManifest) -> String { - let bytes = serde_json::to_vec(manifest).unwrap_or_default(); + // References to the schedule/plan/bundle are bound separately. Excluding + // their digest slots here keeps the immutable identity non-circular while + // still making each artifact commit to the same manifest identity. + let mut identity = manifest.clone(); + identity.campaign_digest = None; + identity.schedule_digest = None; + identity.power_plan_digest = None; + identity.sealed_bundle_digest = None; + let bytes = serde_json::to_vec(&identity).unwrap_or_default(); format!("b3:{}", blake3::hash(&bytes).to_hex()) } @@ -154,15 +233,33 @@ pub fn preflight_frozen_corpus( source_repository: &Path, execute_focused_tests: bool, ) -> CorpusPreflightReport { - let mut errors = validate_frozen_manifest(manifest); - let manifest_valid = errors.is_empty(); + preflight_frozen_corpus_with_options( + manifest, + manifest_directory, + source_repository, + execute_focused_tests, + &CorpusPreflightOptions::default(), + ) +} + +pub fn preflight_frozen_corpus_with_options( + manifest: &FrozenCorpusManifest, + manifest_directory: &Path, + source_repository: &Path, + execute_focused_tests: bool, + options: &CorpusPreflightOptions, +) -> CorpusPreflightReport { + let manifest_errors = validate_frozen_manifest(manifest); + let manifest_valid = manifest_errors.is_empty(); + let mut errors = manifest_errors.clone(); let arms_fixed = manifest.arms == FinalArm::ALL; let source_repository = canonical_or_original(source_repository); let source_sha = git_output(&source_repository, &["rev-parse", "HEAD"]).ok(); let source_clean_before = git_output(&source_repository, &["status", "--short"]) .is_ok_and(|output| output.trim().is_empty()); + let mut source_errors = Vec::new(); if !source_clean_before { - errors.push("source repository is not a clean Git checkout".to_owned()); + source_errors.push("source repository is not a clean Git checkout".to_owned()); } let repository_identities = manifest .tasks @@ -170,26 +267,93 @@ pub fn preflight_frozen_corpus( .map(|task| (&task.repository_url, &task.repository_sha)) .collect::>(); if repository_identities.len() != 1 { - errors + source_errors .push("this preflight requires one source checkout per repository identity".to_owned()); } if let Some(task) = manifest.tasks.first() { if source_sha.as_deref().map(str::trim) != Some(task.repository_sha.as_str()) { - errors.push(format!("source HEAD does not match frozen SHA {}", task.repository_sha)); + source_errors + .push(format!("source HEAD does not match frozen SHA {}", task.repository_sha)); } match git_output(&source_repository, &["remote", "get-url", "origin"]) { Ok(origin) if normalize_git_url(origin.trim()) == normalize_git_url(&task.repository_url) => {} - Ok(origin) => errors.push(format!( - "source origin `{}` does not match frozen repository `{}`", - origin.trim(), - task.repository_url - )), - Err(error) => errors.push(error), + Ok(_) => { + source_errors.push("source origin does not match frozen repository".to_owned()) + } + Err(error) => source_errors.push(error), } } + errors.extend(source_errors.clone()); let manifest_directory = canonical_or_original(manifest_directory); + let (schedule, schedule_digest, mut schedule_errors) = + load_schedule(manifest, &manifest_directory, options); + let (power_plan, power_plan_digest, mut power_plan_errors) = + load_power_plan(manifest, &manifest_directory, options); + let mut power_plan_structurally_valid = manifest.schema != "needle.frozen-corpus/4"; + if let (Some(schedule), Some(plan)) = (schedule.as_ref(), power_plan.as_ref()) { + if let Some(plan_digest) = power_plan_digest.as_deref() { + schedule_errors.extend(schedule.validate(manifest, plan, plan_digest)); + } else { + schedule_errors.push("schedule cannot bind a missing raw power-plan digest".to_owned()); + } + let plan_errors = plan.validate(manifest); + power_plan_structurally_valid = plan_errors.is_empty() && power_plan_errors.is_empty(); + power_plan_errors.extend(plan_errors); + if let Some(plan_digest) = power_plan_digest.as_deref() + && schedule.power_plan_digest != plan_digest + { + schedule_errors.push("schedule power plan digest differs".to_owned()); + } + if plan.synthetic { + power_plan_errors.push( + "synthetic power plan is structurally valid but provider-ineligible".to_owned(), + ); + } + if schedule_errors.is_empty() && power_plan_structurally_valid { + match (schedule_digest.as_deref(), power_plan_digest.as_deref()) { + (Some(schedule_digest), Some(power_plan_digest)) => { + if let Err(projection_errors) = build_launch_plan( + manifest, + schedule, + plan, + schedule_digest, + power_plan_digest, + ) { + schedule_errors.extend(projection_errors); + } + } + _ => schedule_errors.push("launch projection digests are unavailable".to_owned()), + } + } + } + errors.extend(schedule_errors.clone()); + errors.extend(power_plan_errors.clone()); + let bundle_report = if manifest.schema == "needle.frozen-corpus/4" { + validate_sealed_bundle( + manifest, + options.sealed_bundle_index_path.as_deref(), + options.sealed_bundle_root.as_deref(), + ) + } else { + SealedBundleValidationReport { + schema: "needle.sealed-oracle-validation/1".to_owned(), + index_digest: None, + bundle_root: None, + production_material: false, + production_bundle_ready: false, + tasks: Vec::new(), + errors: vec!["legacy corpus has no production sealed bundle".to_owned()], + } + }; + // Legacy v2/v3 manifests are deterministic offline material. Their lack + // of a production sealed bundle is expected and must not become a global + // structural error; v4 supplied-bundle failures remain fail-closed. + if manifest.schema == "needle.frozen-corpus/4" { + errors.extend(bundle_report.errors.clone()); + } + let mut task_reports = Vec::with_capacity(manifest.tasks.len()); for task in &manifest.tasks { let mut task_errors = Vec::new(); @@ -201,46 +365,70 @@ pub fn preflight_frozen_corpus( if !prompt_frozen { task_errors.push("prompt is not a bounded natural task prompt".to_owned()); } - let oracle_path = manifest_directory.join(&task.oracle_path); - let oracle_bytes = fs::read(&oracle_path); + let focused_test_policy_valid = !task.focused_test_policy.identity.trim().is_empty() + && valid_blake3_digest(&task.focused_test_policy.commitment); let mut oracle_bound = false; let mut source_evidence_present = false; - if let Ok(bytes) = oracle_bytes { - let actual_digest = format!("b3:{}", blake3::hash(&bytes).to_hex()); - if actual_digest != task.oracle_digest { - task_errors.push(format!( - "oracle digest mismatch: expected {}, observed {actual_digest}", - task.oracle_digest - )); - } else { - match serde_json::from_slice::(&bytes) { - Ok(oracle) => { - oracle_bound = validate_oracle(task, &oracle, &mut task_errors); - source_evidence_present = - validate_oracle_evidence(&source_repository, &oracle, &mut task_errors); + if manifest.schema == "needle.frozen-corpus/4" { + oracle_bound = bundle_report + .tasks + .iter() + .find(|item| item.task_id == task.id) + .is_some_and(|item| item.bound); + if !focused_test_policy_valid { + task_errors.push("focused-test policy commitment is invalid".to_owned()); + } + if !oracle_bound { + task_errors.push("sealed evaluator material is not bound".to_owned()); + } + } else { + let oracle_path = manifest_directory.join(&task.oracle_path); + let oracle_bytes = read_bounded_file(&oracle_path, crate::MAX_CORPUS_MANIFEST_BYTES); + if let Ok(bytes) = oracle_bytes { + let actual_digest = format!("b3:{}", blake3::hash(&bytes).to_hex()); + if actual_digest != task.oracle_digest { + task_errors.push(format!( + "oracle digest mismatch: expected {}, observed {actual_digest}", + task.oracle_digest + )); + } else { + match serde_json::from_slice::(&bytes) { + Ok(oracle) => { + oracle_bound = validate_oracle(task, &oracle, &mut task_errors); + source_evidence_present = validate_oracle_evidence( + &source_repository, + &oracle, + &mut task_errors, + ); + } + Err(error) => task_errors.push(format!("oracle JSON is invalid: {error}")), } - Err(error) => task_errors.push(format!("oracle JSON is invalid: {error}")), } + } else if let Err(error) = oracle_bytes { + task_errors.push(format!("oracle cannot be read: {error}")); } - } else if let Err(error) = oracle_bytes { - task_errors.push(format!("oracle cannot be read: {error}")); } - - let (focused_test_executed, focused_test_passed) = - if execute_focused_tests && task_errors.is_empty() && source_clean_before { - let passed = execute_focused_test(&source_repository, task, &mut task_errors); - (true, Some(passed)) - } else { - (false, None) - }; + let (focused_test_executed, focused_test_passed) = if execute_focused_tests + && task_errors.is_empty() + && source_clean_before + && manifest.schema != "needle.frozen-corpus/4" + { + let passed = execute_focused_test(&source_repository, task, &mut task_errors); + (true, Some(passed)) + } else { + (false, None) + }; task_reports.push(CorpusTaskPreflight { - task_id: task.id.clone(), + task_id: bounded_label(&task.id), prompt_frozen, oracle_bound, source_evidence_present, focused_test_executed, focused_test_passed, errors: task_errors, + material_class: task.material_class, + focused_test_policy_valid, + sealed_bundle_bound: oracle_bound, }); } let source_clean_after = git_output(&source_repository, &["status", "--short"]) @@ -253,38 +441,165 @@ pub fn preflight_frozen_corpus( } if execute_focused_tests && task_reports.iter().any(|task| task.focused_test_passed != Some(true)) + && manifest.schema != "needle.frozen-corpus/4" { errors.push("one or more focused tests did not execute successfully".to_owned()); } - let calibration_tasks = manifest.tasks.iter().filter(|task| task.split == crate::CorpusSplit::Calibration).count(); - let cost_readiness = - load_cost_readiness(manifest, &manifest_directory, calibration_tasks, &mut errors); + let (cost_readiness, campaign_validation_passed) = load_cost_readiness( + manifest, + &manifest_directory, + calibration_tasks, + schedule.as_ref(), + power_plan.as_ref(), + (schedule_digest.as_deref(), power_plan_digest.as_deref()), + &mut errors, + ); + let campaign_valid = if manifest.schema == "needle.frozen-corpus/4" { + campaign_validation_passed + && cost_readiness.paid_calibration_budget_estimate_microcredits.is_some() + && cost_readiness.full_protocol_campaign_budget_estimate_microcredits.is_some() + } else { + campaign_validation_passed + }; + let provider_inputs_ready = manifest.schema == "needle.frozen-corpus/4" + && manifest_valid + && schedule_errors.is_empty() + && power_plan_structurally_valid + && bundle_report.errors.is_empty() + && bundle_report.production_material + && manifest + .tasks + .iter() + .all(|task| task.material_class == CorpusMaterialClass::ProductionSealed) + && source_errors.is_empty() + && campaign_valid + && cost_readiness.full_protocol_campaign_budget_estimate_microcredits.is_some() + && !plan_is_synthetic(power_plan.as_ref()); + let provider_blockers = vec![ + "v4 provider execution is fail-closed until an isolated executor consumes only ArmLaunch" + .to_owned(), + ]; + let provider_run_ready = false; CorpusPreflightReport { - schema: "needle.corpus-preflight/3".to_owned(), + schema: "needle.corpus-preflight/4".to_owned(), corpus_digest: corpus_digest(manifest), - source_repository: source_repository.display().to_string(), + source_repository: "provided-source-checkout".to_owned(), source_sha: source_sha.map(|value| value.trim().to_owned()), source_clean_before, source_clean_after, manifest_valid, arms_fixed, - campaign_valid: cost_readiness.paid_calibration_budget_estimate_microcredits.is_some(), + campaign_valid, campaign_digest: manifest.campaign_digest.clone(), tasks: task_reports, cost_readiness, - provider_run_ready: false, + provider_inputs_ready, + provider_run_ready, + provider_blockers, errors, + manifest_errors, + schedule_valid: schedule_errors.is_empty(), + schedule_digest, + schedule_errors, + power_plan_valid: power_plan_structurally_valid, + power_plan_digest, + power_plan_errors, + bundle_valid: bundle_report.errors.is_empty(), + bundle_digest: bundle_report.index_digest, + bundle_errors: bundle_report.errors, + source_errors, + } +} + +fn plan_is_synthetic(plan: Option<&PowerPlan>) -> bool { + plan.is_none_or(|plan| plan.synthetic) +} + +fn load_schedule( + manifest: &FrozenCorpusManifest, + directory: &Path, + options: &CorpusPreflightOptions, +) -> (Option, Option, Vec) { + if manifest.schema != "needle.frozen-corpus/4" { + return (None, None, Vec::new()); + } + let path = if let Some(path) = options.schedule_path.as_deref() { + path.to_path_buf() + } else if let Some(path) = manifest.schedule_path.as_deref() { + directory.join(path) + } else { + return (None, None, vec!["schedule is unavailable".to_owned()]); + }; + let bytes = match read_bounded_file(&path, crate::schedule::MAX_SCHEDULE_BYTES) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::InvalidData => { + return (None, None, vec!["schedule exceeds bounded byte limit".to_owned()]); + } + Err(_) => return (None, None, vec!["schedule cannot be read".to_owned()]), + }; + let digest = format!("b3:{}", blake3::hash(&bytes).to_hex()); + let mut errors = Vec::new(); + if manifest.schedule_digest.as_deref() != Some(digest.as_str()) { + errors.push("schedule digest differs from manifest".to_owned()); } + let schedule = match serde_json::from_slice::(&bytes) { + Ok(value) => Some(value), + Err(error) => { + errors.push(format!("schedule JSON is invalid: {error}")); + None + } + }; + (schedule, Some(digest), errors) +} + +fn load_power_plan( + manifest: &FrozenCorpusManifest, + directory: &Path, + options: &CorpusPreflightOptions, +) -> (Option, Option, Vec) { + if manifest.schema != "needle.frozen-corpus/4" { + return (None, None, Vec::new()); + } + let path = if let Some(path) = options.power_plan_path.as_deref() { + path.to_path_buf() + } else if let Some(path) = manifest.power_plan_path.as_deref() { + directory.join(path) + } else { + return (None, None, vec!["power plan is unavailable".to_owned()]); + }; + let bytes = match read_bounded_file(&path, crate::schedule::MAX_SCHEDULE_BYTES) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::InvalidData => { + return (None, None, vec!["power plan exceeds bounded byte limit".to_owned()]); + } + Err(_) => return (None, None, vec!["power plan cannot be read".to_owned()]), + }; + let digest = format!("b3:{}", blake3::hash(&bytes).to_hex()); + let mut errors = Vec::new(); + if manifest.power_plan_digest.as_deref() != Some(digest.as_str()) { + errors.push("power plan digest differs from manifest".to_owned()); + } + let plan = match serde_json::from_slice::(&bytes) { + Ok(value) => Some(value), + Err(error) => { + errors.push(format!("power plan JSON is invalid: {error}")); + None + } + }; + (plan, Some(digest), errors) } fn load_cost_readiness( manifest: &FrozenCorpusManifest, manifest_directory: &Path, calibration_tasks: usize, + schedule: Option<&CorpusSchedule>, + power_plan: Option<&PowerPlan>, + (schedule_digest, power_plan_digest): (Option<&str>, Option<&str>), errors: &mut Vec, -) -> CampaignCostReadiness { +) -> (CampaignCostReadiness, bool) { let mut readiness = CampaignCostReadiness { next_pilot_observations: 0, next_pilot_budget_estimate_microcredits: None, @@ -310,11 +625,11 @@ fn load_cost_readiness( ], }; let path = manifest_directory.join(&manifest.cost_model_path); - let bytes = match fs::read(&path) { + let bytes = match read_bounded_file(&path, crate::MAX_CORPUS_MANIFEST_BYTES) { Ok(bytes) => bytes, Err(error) => { errors.push(format!("cost model cannot be read: {error}")); - return readiness; + return (readiness, false); } }; let digest = format!("b3:{}", blake3::hash(&bytes).to_hex()); @@ -323,13 +638,13 @@ fn load_cost_readiness( "cost model digest mismatch: expected {}, observed {digest}", manifest.cost_model_digest )); - return readiness; + return (readiness, false); } let model: CampaignCostModel = match serde_json::from_slice(&bytes) { Ok(model) => model, Err(error) => { errors.push(format!("cost model JSON is invalid: {error}")); - return readiness; + return (readiness, false); } }; let configured = model.arm_estimates.iter().map(|item| item.arm).collect::>(); @@ -347,7 +662,7 @@ fn load_cost_readiness( }) { errors.push("cost model is incomplete or internally inconsistent".to_owned()); - return readiness; + return (readiness, false); } let per_task = model.arm_estimates.iter().try_fold(0_u64, |total, estimate| { total.checked_add(estimate.microcredits_per_observation) @@ -360,27 +675,129 @@ fn load_cost_readiness( readiness.assumptions = model.arm_estimates.iter().map(|item| format!("{:?}: {}", item.arm, item.basis)).collect(); load_minimal_pilot(manifest, manifest_directory, &model, &mut readiness, errors); - load_multi_task_campaign(manifest, manifest_directory, &model, &mut readiness, errors); - readiness + let schedule_context = schedule + .zip(power_plan) + .zip(schedule_digest) + .map(|((schedule, power_plan), schedule_digest)| (schedule, power_plan, schedule_digest)); + let campaign_reserve = load_multi_task_campaign( + manifest, + manifest_directory, + &model, + schedule_context, + &mut readiness, + errors, + ); + if manifest.schema == "needle.frozen-corpus/4" { + match (schedule, power_plan, schedule_digest, power_plan_digest) { + ( + Some(schedule), + Some(plan), + Some(raw_schedule_digest), + Some(raw_power_plan_digest), + ) if schedule.entries.len() <= crate::schedule::MAX_SCHEDULE_ENTRIES + && schedule.validate(manifest, plan, raw_power_plan_digest).is_empty() + && valid_blake3_digest(raw_schedule_digest) => + { + let mut total = 0_u64; + let mut holdout = 0_usize; + let mut budget_valid = true; + for entry in &schedule.entries { + let Some(estimate) = model + .arm_estimates + .iter() + .find(|estimate| estimate.arm == entry.arm) + .map(|estimate| estimate.microcredits_per_observation) + else { + errors.push("schedule arm is missing from cost model".to_owned()); + budget_valid = false; + break; + }; + let Some(next) = total.checked_add(estimate) else { + errors.push("full protocol campaign budget overflowed".to_owned()); + budget_valid = false; + break; + }; + total = next; + if entry.arm == FinalArm::NeedleMiss { + let Some(reserve) = campaign_reserve.as_ref() else { + errors.push( + "full protocol campaign budget is unavailable without a valid v2 campaign" + .to_owned(), + ); + budget_valid = false; + break; + }; + let Some(main_reserve) = reserve + .main_turn_microcredits + .checked_mul(reserve.extra_main_turns_per_needle_observation as u64) + else { + errors.push("full protocol campaign budget overflowed".to_owned()); + budget_valid = false; + break; + }; + let Some(worker_reserve) = reserve + .worker_microcredits + .checked_mul(reserve.extra_workers_per_needle_observation as u64) + else { + errors.push("full protocol campaign budget overflowed".to_owned()); + budget_valid = false; + break; + }; + let Some(next) = total.checked_add(main_reserve) else { + errors.push("full protocol campaign budget overflowed".to_owned()); + budget_valid = false; + break; + }; + let Some(next) = next.checked_add(worker_reserve) else { + errors.push("full protocol campaign budget overflowed".to_owned()); + budget_valid = false; + break; + }; + total = next; + } + if entry.split == crate::CorpusSplit::Holdout { + holdout = holdout.saturating_add(1); + } + } + if budget_valid { + readiness.powered_holdout_observations = Some(holdout); + readiness.full_protocol_campaign_budget_estimate_microcredits = Some(total); + readiness.assumptions.push( + "full campaign budget is the checked schedule arm-cost sum plus checked main-turn and worker reserves for every scheduled NeedleMiss observation; no provider run is implied" + .to_owned(), + ); + } + } + _ => errors.push( + "full protocol campaign budget is unavailable without a validated schedule" + .to_owned(), + ), + } + } + (readiness, campaign_reserve.is_some()) } fn load_multi_task_campaign( manifest: &FrozenCorpusManifest, manifest_directory: &Path, cost_model: &CampaignCostModel, + schedule_context: Option<(&CorpusSchedule, &PowerPlan, &str)>, readiness: &mut CampaignCostReadiness, errors: &mut Vec, -) { +) -> Option { let (Some(campaign_path), Some(expected_digest)) = (&manifest.campaign_path, &manifest.campaign_digest) else { - return; + return None; }; - let bytes = match fs::read(manifest_directory.join(campaign_path)) { + let bytes = match read_bounded_file( + &manifest_directory.join(campaign_path), + crate::MAX_CORPUS_MANIFEST_BYTES, + ) { Ok(bytes) => bytes, Err(error) => { errors.push(format!("multi-task campaign cannot be read: {error}")); - return; + return None; } }; let digest = format!("b3:{}", blake3::hash(&bytes).to_hex()); @@ -388,13 +805,13 @@ fn load_multi_task_campaign( errors.push(format!( "multi-task campaign digest mismatch: expected {expected_digest}, observed {digest}" )); - return; + return None; } let campaign: MultiTaskCampaign = match serde_json::from_slice(&bytes) { Ok(campaign) => campaign, Err(error) => { errors.push(format!("multi-task campaign JSON is invalid: {error}")); - return; + return None; } }; let paid = [FinalArm::FrontierDirect, FinalArm::NeedleMiss]; @@ -405,6 +822,17 @@ fn load_multi_task_campaign( FinalArm::RelevantMutation, ]; let deferred = [FinalArm::NativeSubagent, FinalArm::Escalation]; + let is_v2 = campaign.schema == "needle.multi-task-campaign/2"; + if is_v2 { + let Some((_, power_plan, _)) = schedule_context else { + errors.push("v2 campaign requires a validated schedule and power plan".to_owned()); + return None; + }; + if power_plan.campaign_digest != campaign_commitment(&campaign) { + errors.push("power plan campaign commitment differs from campaign".to_owned()); + return None; + } + } let task_ids = campaign.task_ids.iter().map(String::as_str).collect::>(); let calibration_tasks = manifest .tasks @@ -425,15 +853,22 @@ fn load_multi_task_campaign( .chain(&campaign.deferred_diagnostic_arms) .copied() .collect::>(); - if campaign.schema != "needle.multi-task-campaign/1" - || campaign.task_ids.len() != task_ids.len() - || task_ids != expected_task_ids + let campaign_schedule_matches = + !is_v2 || schedule_context.is_some_and(|(_, _, digest)| campaign.schedule_digest == digest); + let legacy_task_authority_matches = + is_v2 || (campaign.task_ids.len() == task_ids.len() && task_ids == expected_task_ids); + if !matches!( + campaign.schema.as_str(), + "needle.multi-task-campaign/1" | "needle.multi-task-campaign/2" + ) || !campaign_schedule_matches + || !legacy_task_authority_matches || routes != expected_routes || campaign.paid_arms != paid || campaign.offline_cache_arms != offline || campaign.deferred_diagnostic_arms != deferred || configured_arms != FinalArm::ALL.into_iter().collect::>() - || campaign.repetitions_per_task != 1 + || (!is_v2 && campaign.repetitions_per_task != 1) + || (is_v2 && (!campaign.task_ids.is_empty() || campaign.repetitions_per_task != 0)) || campaign.automatic_retries || campaign.statistical_claim || campaign.bootstrap_resamples < 1_000 @@ -448,11 +883,11 @@ fn load_multi_task_campaign( "multi-task campaign must partition the calibration arms into paid economics, offline cache validation and deferred diagnostics with one no-retry repetition" .to_owned(), ); - return; + return None; } - let task_count = campaign.task_ids.len(); - let repetitions = campaign.repetitions_per_task as usize; + let task_count = if is_v2 { calibration_tasks.len() } else { campaign.task_ids.len() }; + let repetitions = if is_v2 { 1 } else { campaign.repetitions_per_task as usize }; readiness.paid_calibration_observations = task_count * campaign.paid_arms.len() * repetitions; readiness.offline_cache_validation_observations = task_count * campaign.offline_cache_arms.len() * repetitions; @@ -488,7 +923,7 @@ fn load_multi_task_campaign( }); if readiness.paid_calibration_budget_estimate_microcredits.is_none() { errors.push("multi-task paid calibration budget estimate is unavailable".to_owned()); - return; + return None; } readiness.assumptions.push(format!( "multi-task campaign: {} paid economics, {} offline cache-validation and {} deferred diagnostic observations", @@ -496,6 +931,7 @@ fn load_multi_task_campaign( readiness.offline_cache_validation_observations, readiness.deferred_diagnostic_observations )); + Some(campaign.budget_reserve) } fn load_minimal_pilot( @@ -506,7 +942,7 @@ fn load_minimal_pilot( errors: &mut Vec, ) { let path = manifest_directory.join(&manifest.next_pilot_path); - let bytes = match fs::read(&path) { + let bytes = match read_bounded_file(&path, crate::MAX_CORPUS_MANIFEST_BYTES) { Ok(bytes) => bytes, Err(error) => { errors.push(format!("minimal pilot cannot be read: {error}")); @@ -719,6 +1155,16 @@ fn normalize_git_url(value: &str) -> String { value.trim().trim_end_matches('/').trim_end_matches(".git").to_ascii_lowercase() } +fn valid_blake3_digest(value: &str) -> bool { + value.len() == 67 + && value.starts_with("b3:") + && value[3..].bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn bounded_label(value: &str) -> String { + value.chars().take(256).collect() +} + #[cfg(test)] mod tests { use super::*; @@ -729,13 +1175,15 @@ mod tests { let directory = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../benchmarks/corpus/router-cache"); let manifest: FrozenCorpusManifest = serde_json::from_slice( - &fs::read(directory.join("manifest.json")).expect("current manifest"), + &fs::read(directory.join("legacy-offline-manifest.json")).expect("legacy manifest"), ) - .expect("valid current manifest"); + .expect("valid legacy manifest"); let mut errors = Vec::new(); - let readiness = load_cost_readiness(&manifest, &directory, 2, &mut errors); + let (readiness, campaign_valid) = + load_cost_readiness(&manifest, &directory, 2, None, None, (None, None), &mut errors); assert!(errors.is_empty(), "{errors:?}"); + assert!(campaign_valid); assert_eq!(readiness.paid_calibration_observations, 4); assert_eq!(readiness.offline_cache_validation_observations, 8); assert_eq!(readiness.deferred_diagnostic_observations, 4); @@ -761,6 +1209,12 @@ mod tests { "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), campaign_path: None, campaign_digest: None, + schedule_path: None, + schedule_digest: None, + power_plan_path: None, + power_plan_digest: None, + sealed_bundle_schema: None, + sealed_bundle_digest: None, tasks: vec![CorpusTask { id: "missing".to_owned(), route: BenchmarkRoute::LocateImplementation, @@ -769,6 +1223,9 @@ mod tests { repository_sha: "0123456789abcdef0123456789abcdef01234567".to_owned(), prompt: "Locate the implementation and identify one focused behavioral test." .to_owned(), + material_class: CorpusMaterialClass::Legacy, + focused_test_policy: crate::FocusedTestPolicyRef::default(), + oracle_schema: String::new(), oracle_path: "oracles/missing.json".to_owned(), oracle_digest: "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), @@ -795,4 +1252,29 @@ mod tests { report.cost_readiness.full_protocol_calibration_budget_estimate_microcredits.is_none() ); } + + #[test] + fn preflight_exit_distinguishes_expected_nonreadiness_from_invalid_supplied_bundle() { + let directory = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../benchmarks/corpus/router-cache"); + let manifest: FrozenCorpusManifest = + serde_json::from_slice(&fs::read(directory.join("manifest.json")).expect("manifest")) + .expect("manifest JSON"); + let source = directory.join("missing-source-checkout"); + let report = preflight_frozen_corpus(&manifest, &directory, &source, false); + assert!(report.campaign_valid); + assert_eq!( + report.cost_readiness.full_protocol_campaign_budget_estimate_microcredits, + Some(172_157_430) + ); + assert!(!report.provider_inputs_ready); + assert!(!report.provider_run_ready); + assert!(!preflight_exit_is_failure(&report, &CorpusPreflightOptions::default(), false)); + let invalid_options = CorpusPreflightOptions { + sealed_bundle_index_path: Some(directory.join("missing-index.json")), + sealed_bundle_root: Some(directory.join("missing-bundle")), + ..CorpusPreflightOptions::default() + }; + assert!(preflight_exit_is_failure(&report, &invalid_options, false)); + } } diff --git a/crates/needle-bench/src/final_gate.rs b/crates/needle-bench/src/final_gate.rs index cf0bc03..7dd32a5 100644 --- a/crates/needle-bench/src/final_gate.rs +++ b/crates/needle-bench/src/final_gate.rs @@ -2,6 +2,10 @@ use serde::{Deserialize, Serialize}; use statrs::distribution::{ContinuousCDF, Normal}; use std::collections::{BTreeMap, BTreeSet}; +pub const MAX_CORPUS_TASKS: usize = 512; +pub const MAX_CORPUS_MANIFEST_BYTES: usize = 1024 * 1024; +const MAX_CORPUS_IDENTIFIER_BYTES: usize = 256; + #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum BenchmarkRoute { @@ -16,6 +20,33 @@ pub enum CorpusSplit { Holdout, } +/// Classification carried by a public corpus task. Synthetic and legacy +/// material is useful for deterministic/offline fixtures only and can never +/// make a provider campaign ready. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CorpusMaterialClass { + ProductionSealed, + Synthetic, + Legacy, +} + +impl Default for CorpusMaterialClass { + fn default() -> Self { + Self::Legacy + } +} + +/// A bounded, digest-bound focused-test policy. The argv is intentionally +/// not part of a public manifest or launch projection; it is resolved by the +/// evaluator-owned sealed contract. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FocusedTestPolicyRef { + pub identity: String, + pub commitment: String, +} + #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum FinalArm { @@ -51,9 +82,22 @@ pub struct CorpusTask { pub repository_url: String, pub repository_sha: String, pub prompt: String, - pub oracle_path: String, + #[serde(default)] + pub material_class: CorpusMaterialClass, + #[serde(default)] + pub focused_test_policy: FocusedTestPolicyRef, + #[serde(default)] + pub oracle_schema: String, pub oracle_digest: String, + + // These fields are retained only so archived v2/v3 readers and existing + // offline synthetic demos can be decoded. They are never serialized in + // the v4 public manifest and are rejected for provider execution. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub oracle_path: String, + #[serde(default, skip_serializing_if = "String::is_empty")] pub test_identifier: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub focused_command: Vec, } @@ -71,6 +115,20 @@ pub struct FrozenCorpusManifest { pub campaign_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub campaign_digest: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule_digest: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub power_plan_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub power_plan_digest: Option, + /// Commitment to the evaluator contract/index. It is deliberately a + /// digest only; no external bundle path or location is public. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sealed_bundle_schema: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sealed_bundle_digest: Option, pub tasks: Vec, } @@ -168,6 +226,12 @@ pub fn evaluate_final_gate( bootstrap_seed: u64, ) -> FinalGateReport { let mut manifest_errors = validate_frozen_manifest(manifest); + if manifest.schema == "needle.frozen-corpus/4" { + manifest_errors.push( + "v4 final gate is unavailable until schedule-bound exact observations are integrated" + .to_owned(), + ); + } manifest_errors.extend(validate_observations(manifest, observations)); let manifest_valid = manifest_errors.is_empty(); let routes = [BenchmarkRoute::LocateImplementation, BenchmarkRoute::TraceStateFlow] @@ -190,6 +254,10 @@ pub fn evaluate_final_gate( pub fn validate_frozen_manifest(manifest: &FrozenCorpusManifest) -> Vec { let mut errors = Vec::new(); + if manifest.schema == "needle.frozen-corpus/4" { + validate_v4_manifest(manifest, &mut errors); + return errors; + } if !matches!(manifest.schema.as_str(), "needle.frozen-corpus/2" | "needle.frozen-corpus/3") { errors.push("unsupported frozen corpus schema".to_owned()); } @@ -228,7 +296,7 @@ pub fn validate_frozen_manifest(manifest: &FrozenCorpusManifest) -> Vec let mut tasks = BTreeMap::new(); for task in &manifest.tasks { if tasks.insert(task.id.as_str(), task).is_some() { - errors.push(format!("duplicate corpus task `{}`", task.id)); + errors.push("duplicate corpus task identifier".to_owned()); } if task.repository_url.is_empty() || task.repository_sha.len() != 40 @@ -241,7 +309,7 @@ pub fn validate_frozen_manifest(manifest: &FrozenCorpusManifest) -> Vec || task.test_identifier.trim().is_empty() || task.focused_command.is_empty() { - errors.push(format!("corpus task `{}` is incomplete", task.id)); + errors.push("corpus task metadata is incomplete or exceeds bounds".to_owned()); } let expected_command = [ "cargo", @@ -270,6 +338,99 @@ pub fn validate_frozen_manifest(manifest: &FrozenCorpusManifest) -> Vec errors } +fn validate_v4_manifest(manifest: &FrozenCorpusManifest, errors: &mut Vec) { + if manifest.frozen_unix_ms == 0 { + errors.push("frozen corpus timestamp is missing".to_owned()); + } + let configured_arms = manifest.arms.iter().copied().collect::>(); + let required_arms = FinalArm::ALL.into_iter().collect::>(); + if manifest.arms.len() != configured_arms.len() || configured_arms != required_arms { + errors.push("frozen corpus must configure every final arm exactly once".to_owned()); + } + for (label, path, digest) in [ + ( + "cost model", + Some(manifest.cost_model_path.as_str()), + Some(manifest.cost_model_digest.as_str()), + ), + ( + "minimal pilot", + Some(manifest.next_pilot_path.as_str()), + Some(manifest.next_pilot_digest.as_str()), + ), + ] { + if path.is_none_or(|value| !safe_relative_json_path(value)) + || digest.is_none_or(|value| !valid_blake3_digest(value)) + { + errors.push(format!("frozen corpus {label} reference is invalid")); + } + } + match (manifest.campaign_path.as_deref(), manifest.campaign_digest.as_deref()) { + (Some(path), Some(digest)) + if safe_relative_json_path(path) && valid_blake3_digest(digest) => {} + _ => errors.push("frozen corpus v4 requires a valid campaign reference".to_owned()), + } + match (manifest.schedule_path.as_deref(), manifest.schedule_digest.as_deref()) { + (Some(path), Some(digest)) + if safe_relative_json_path(path) && valid_blake3_digest(digest) => {} + _ => errors.push("frozen corpus v4 requires a valid schedule reference".to_owned()), + } + match (manifest.power_plan_path.as_deref(), manifest.power_plan_digest.as_deref()) { + (Some(path), Some(digest)) + if safe_relative_json_path(path) && valid_blake3_digest(digest) => {} + _ => errors.push("frozen corpus v4 requires a valid power-plan reference".to_owned()), + } + match (manifest.sealed_bundle_schema.as_deref(), manifest.sealed_bundle_digest.as_deref()) { + (Some(schema), Some(digest)) + if schema == "needle.sealed-oracle-index/1" && valid_blake3_digest(digest) => {} + _ => errors.push("frozen corpus v4 requires a sealed evaluator commitment".to_owned()), + } + if manifest.tasks.is_empty() || manifest.tasks.len() > MAX_CORPUS_TASKS { + errors.push("frozen corpus task count is out of bounds".to_owned()); + } + let mut tasks = BTreeSet::new(); + for task in &manifest.tasks { + if !tasks.insert(task.id.as_str()) { + errors.push("duplicate corpus task identifier".to_owned()); + } + if task.id.len() > MAX_CORPUS_IDENTIFIER_BYTES + || task.repository_url.len() > 2_048 + || task.prompt.len() > 4_000 + || task.focused_test_policy.identity.len() > MAX_CORPUS_IDENTIFIER_BYTES + || task.oracle_schema.len() > MAX_CORPUS_IDENTIFIER_BYTES + || task.id.trim().is_empty() + || task.repository_url.trim().is_empty() + || task.repository_sha.len() != 40 + || !task.repository_sha.bytes().all(|byte| byte.is_ascii_hexdigit()) + || task.prompt.trim().len() < 40 + || task.prompt.len() > 4_000 + || task.prompt.contains("@@need") + || task.focused_test_policy.identity.trim().is_empty() + || !valid_blake3_digest(&task.focused_test_policy.commitment) + || task.oracle_schema != "needle.sealed-oracle/1" + || !valid_blake3_digest(&task.oracle_digest) + { + errors.push("corpus task metadata is incomplete or exceeds bounds".to_owned()); + } + if !task.oracle_path.is_empty() + || !task.test_identifier.is_empty() + || !task.focused_command.is_empty() + { + errors.push("v4 corpus task contains legacy answer-bearing fields".to_owned()); + } + if task.material_class == CorpusMaterialClass::Legacy { + errors.push("corpus task is legacy and provider-ineligible".to_owned()); + } + } + for route in [BenchmarkRoute::LocateImplementation, BenchmarkRoute::TraceStateFlow] { + for split in [CorpusSplit::Calibration, CorpusSplit::Holdout] { + if !manifest.tasks.iter().any(|task| task.route == route && task.split == split) { + errors.push(format!("corpus is missing {route:?}/{split:?} tasks")); + } + } + } +} + fn validate_observations( manifest: &FrozenCorpusManifest, observations: &[FinalObservation], @@ -311,7 +472,7 @@ fn validate_observations( fn valid_blake3_digest(value: &str) -> bool { value.len() == 67 && value.starts_with("b3:") - && value[3..].bytes().all(|byte| byte.is_ascii_hexdigit()) + && value[3..].bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } fn safe_relative_json_path(value: &str) -> bool { @@ -507,6 +668,9 @@ mod tests { prompt: "Locate the implementation and provide a focused test for this behavior." .to_owned(), + material_class: CorpusMaterialClass::Legacy, + focused_test_policy: FocusedTestPolicyRef::default(), + oracle_schema: String::new(), oracle_path: format!("oracles/{route:?}-{split:?}.json"), oracle_digest: "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" @@ -540,6 +704,12 @@ mod tests { "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), campaign_path: None, campaign_digest: None, + schedule_path: None, + schedule_digest: None, + power_plan_path: None, + power_plan_digest: None, + sealed_bundle_schema: None, + sealed_bundle_digest: None, tasks, } } @@ -668,6 +838,42 @@ mod tests { assert!(validate_frozen_manifest(&manifest).is_empty()); } + #[test] + fn legacy_answer_fields_round_trip_but_v4_rejects_them() { + let legacy = manifest(); + let encoded = serde_json::to_vec(&legacy).expect("legacy manifest serializes"); + let decoded: FrozenCorpusManifest = + serde_json::from_slice(&encoded).expect("legacy manifest round-trips"); + assert_eq!(decoded, legacy); + + let mut v4 = legacy; + v4.schema = "needle.frozen-corpus/4".to_owned(); + v4.tasks.iter_mut().for_each(|task| { + task.material_class = CorpusMaterialClass::Synthetic; + task.oracle_path.clear(); + task.test_identifier.clear(); + task.focused_command.clear(); + }); + let mut contaminated = v4.clone(); + contaminated.tasks[0].oracle_path = "legacy.json".to_owned(); + let errors = validate_frozen_manifest(&contaminated); + assert!(errors.iter().any(|error| error.contains("legacy answer-bearing"))); + } + + #[test] + fn v4_final_gate_is_explicitly_fail_closed() { + let mut manifest = manifest(); + manifest.schema = "needle.frozen-corpus/4".to_owned(); + let report = evaluate_final_gate(&manifest, &[], 1_000, 1); + assert!(!report.passed); + assert!( + report + .manifest_errors + .iter() + .any(|error| error.contains("schedule-bound exact observations")) + ); + } + #[test] fn power_estimate_uses_only_observed_calibration_variance() { let estimate = estimate_required_pairs(&[0.70, 0.76, 0.82, 0.73]).unwrap(); diff --git a/crates/needle-bench/src/lib.rs b/crates/needle-bench/src/lib.rs index 5538874..8d61f1f 100644 --- a/crates/needle-bench/src/lib.rs +++ b/crates/needle-bench/src/lib.rs @@ -6,6 +6,8 @@ mod corpus; mod final_gate; mod minimal_pilot; mod product; +mod schedule; +mod sealed_oracle; mod shadow_replay; pub use artifact_cache_replay::*; @@ -14,6 +16,8 @@ pub use corpus::*; pub use final_gate::*; pub use minimal_pilot::*; pub use product::*; +pub use schedule::*; +pub use sealed_oracle::*; pub use shadow_replay::*; use needle_core::{ContinuationEnvelope, Digest, NeedRequest}; @@ -21,10 +25,28 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use std::collections::BTreeMap; use std::fs; -use std::io; +use std::io::{self, Read}; use std::path::{Path, PathBuf}; use thiserror::Error; +/// Read at most `max_bytes` from an untrusted file without first allocating +/// the file's full advertised size. The one-byte over-read distinguishes an +/// exact-bound file from an oversized file even when metadata changes between +/// the precheck and the read. +pub fn read_bounded_file(path: &Path, max_bytes: usize) -> io::Result> { + let metadata = fs::metadata(path)?; + if metadata.len() > max_bytes as u64 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "file exceeds bounded byte limit")); + } + let mut file = fs::File::open(path)?; + let mut bytes = Vec::with_capacity(metadata.len().min(max_bytes as u64) as usize); + file.by_ref().take(max_bytes as u64 + 1).read_to_end(&mut bytes)?; + if bytes.len() > max_bytes { + return Err(io::Error::new(io::ErrorKind::InvalidData, "file exceeds bounded byte limit")); + } + Ok(bytes) +} + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] #[serde(rename_all = "UPPERCASE")] pub enum ExperimentArm { diff --git a/crates/needle-bench/src/schedule.rs b/crates/needle-bench/src/schedule.rs new file mode 100644 index 0000000..29cbc21 --- /dev/null +++ b/crates/needle-bench/src/schedule.rs @@ -0,0 +1,635 @@ +use crate::{ + BenchmarkRoute, CorpusMaterialClass, CorpusSplit, CorpusTask, FinalArm, FrozenCorpusManifest, + corpus_digest, +}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +pub const CORPUS_SCHEDULE_SCHEMA: &str = "needle.corpus-schedule/1"; +pub const POWER_PLAN_SCHEMA: &str = "needle.power-plan/1"; +pub const ARM_LAUNCH_SCHEMA: &str = "needle.arm-launch/1"; +pub const MAX_POWER_PLAN_PAIRS: u32 = 10_000; +pub const MAX_SCHEDULE_ENTRIES: usize = 100_000; +pub const MAX_SCHEDULE_BYTES: usize = 4 * 1024 * 1024; +const MAX_IDENTIFIER_BYTES: usize = 256; +const MAX_PROMPT_BYTES: usize = 4_000; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScheduleEntry { + pub task_id: String, + pub route: BenchmarkRoute, + pub split: CorpusSplit, + pub arm: FinalArm, + pub repetition: u32, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RouteRepetitions { + pub route: BenchmarkRoute, + pub repetitions: u32, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PowerRoutePlan { + pub route: BenchmarkRoute, + pub baseline_arm: FinalArm, + pub treatment_arm: FinalArm, + pub pair_key: String, + pub observed_log_ratio_mean: f64, + pub observed_log_ratio_stddev: f64, + pub required_pairs: u32, +} + +/// Immutable, evaluator-independent count contract. Issue #7 owns the +/// statistical production of a real plan; this type only binds the resulting +/// counts and provenance to a manifest. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PowerPlan { + pub schema: String, + pub plan_id: String, + pub manifest_digest: String, + pub campaign_digest: String, + pub calibration_input_digest: String, + pub estimator_revision: String, + pub alpha_basis_points: u16, + pub target_power_basis_points: u16, + pub routes: Vec, + pub validated: bool, + pub synthetic: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PowerPlanReference { + pub schema: String, + pub digest: String, +} + +impl PowerPlanReference { + pub fn from_plan_bytes(bytes: &[u8]) -> Self { + Self { schema: POWER_PLAN_SCHEMA.to_owned(), digest: raw_digest(bytes) } + } +} + +impl PowerPlan { + pub fn validate(&self, manifest: &FrozenCorpusManifest) -> Vec { + let mut errors = Vec::new(); + if self.schema != POWER_PLAN_SCHEMA { + errors.push("power plan schema is unsupported".to_owned()); + } + if self.plan_id.trim().is_empty() || self.plan_id.len() > MAX_IDENTIFIER_BYTES { + errors.push("power plan id is missing".to_owned()); + } + if self.manifest_digest != corpus_digest(manifest) { + errors.push("power plan manifest digest differs".to_owned()); + } + if !valid_digest(&self.campaign_digest) { + errors.push("power plan campaign digest is invalid".to_owned()); + } + if !valid_digest(&self.calibration_input_digest) { + errors.push("power plan calibration-input digest is invalid".to_owned()); + } + if self.estimator_revision.trim().is_empty() || self.estimator_revision.len() > 128 { + errors.push("power plan estimator revision is invalid".to_owned()); + } + if self.alpha_basis_points != 500 || self.target_power_basis_points != 9_000 { + errors.push("power plan alpha/target-power contract differs".to_owned()); + } + let expected_routes = + [BenchmarkRoute::LocateImplementation, BenchmarkRoute::TraceStateFlow]; + if self.routes.len() != expected_routes.len() + || self.routes.iter().map(|item| item.route).ne(expected_routes) + { + errors.push("power plan routes must be canonical and complete".to_owned()); + } + let mut seen_routes = BTreeSet::new(); + for route in &self.routes { + if !seen_routes.insert(route.route) { + errors.push("power plan contains duplicate routes".to_owned()); + } + if route.baseline_arm != FinalArm::FrontierDirect + || route.treatment_arm != FinalArm::NeedleMiss + || route.pair_key != "task_id:repetition" + || !route.observed_log_ratio_mean.is_finite() + || route.observed_log_ratio_mean >= 0.0 + || route.observed_log_ratio_mean < -100.0 + || !route.observed_log_ratio_stddev.is_finite() + || route.observed_log_ratio_stddev <= 0.0 + || route.observed_log_ratio_stddev > 100.0 + || route.required_pairs == 0 + || route.required_pairs > MAX_POWER_PLAN_PAIRS + { + errors + .push(format!("power plan route {:?} has invalid bounded values", route.route)); + } + } + if !self.validated { + errors.push("power plan has not been validated".to_owned()); + } + errors + } + + pub fn required_pairs(&self, route: BenchmarkRoute) -> Option { + self.routes.iter().find(|item| item.route == route).map(|item| item.required_pairs) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CorpusSchedule { + pub schema: String, + pub manifest_digest: String, + pub power_plan_digest: String, + pub automatic_retries: bool, + pub entries: Vec, +} + +impl CorpusSchedule { + pub fn validate( + &self, + manifest: &FrozenCorpusManifest, + power_plan: &PowerPlan, + raw_power_plan_digest: &str, + ) -> Vec { + let mut errors = Vec::new(); + if self.schema != CORPUS_SCHEDULE_SCHEMA { + errors.push("corpus schedule schema is unsupported".to_owned()); + } + if self.manifest_digest != corpus_digest(manifest) { + errors.push("schedule manifest digest differs".to_owned()); + } + if self.power_plan_digest != raw_power_plan_digest || !valid_digest(raw_power_plan_digest) { + errors.push("schedule power plan digest differs from raw power-plan bytes".to_owned()); + } + if self.automatic_retries { + errors.push("automatic retries must be disabled".to_owned()); + } + + if self.entries.len() > MAX_SCHEDULE_ENTRIES { + errors.push("schedule entry count exceeds bounded maximum".to_owned()); + return errors; + } + if self.entries.iter().any(|entry| { + entry.task_id.len() > MAX_IDENTIFIER_BYTES || entry.task_id.trim().is_empty() + }) { + errors.push("schedule task identifier exceeds bounded length".to_owned()); + return errors; + } + validate_entries_in_manifest_order(self, manifest, power_plan, &mut errors); + errors + } +} + +/// Bounded projection handed to a model/worker runner. No answer, oracle +/// bytes, focused argv, or external bundle location can be represented here. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArmLaunch { + pub schema: String, + pub manifest_digest: String, + pub schedule_digest: String, + pub power_plan_digest: String, + pub task_id: String, + pub route: BenchmarkRoute, + pub split: CorpusSplit, + pub repository_sha: String, + pub prompt: String, + pub arm: FinalArm, + pub repetition: u32, + pub focused_test_policy_identity: String, + pub focused_test_policy_commitment: String, +} + +impl ArmLaunch { + fn from_task_with_digests( + task: &CorpusTask, + arm: FinalArm, + repetition: u32, + manifest_digest: &str, + schedule_digest: &str, + power_plan_digest: &str, + ) -> Result { + if !valid_digest(manifest_digest) + || !valid_digest(schedule_digest) + || !valid_digest(power_plan_digest) + { + return Err("launch artifact digests are missing or invalid".to_owned()); + } + if task.material_class == CorpusMaterialClass::Legacy { + return Err("legacy task cannot be projected for provider execution".to_owned()); + } + if task.id.trim().is_empty() + || task.repository_sha.len() != 40 + || !task.repository_sha.bytes().all(|byte| byte.is_ascii_hexdigit()) + || task.prompt.trim().is_empty() + || task.id.len() > MAX_IDENTIFIER_BYTES + || task.prompt.len() > MAX_PROMPT_BYTES + || task.focused_test_policy.identity.trim().is_empty() + || !valid_digest(&task.focused_test_policy.commitment) + { + return Err("task has incomplete bounded launch metadata".to_owned()); + } + Ok(Self { + schema: ARM_LAUNCH_SCHEMA.to_owned(), + manifest_digest: manifest_digest.to_owned(), + schedule_digest: schedule_digest.to_owned(), + power_plan_digest: power_plan_digest.to_owned(), + task_id: task.id.clone(), + route: task.route, + split: task.split, + repository_sha: task.repository_sha.clone(), + prompt: task.prompt.clone(), + arm, + repetition, + focused_test_policy_identity: task.focused_test_policy.identity.clone(), + focused_test_policy_commitment: task.focused_test_policy.commitment.clone(), + }) + } + + pub fn serialization_is_bounded(&self) -> bool { + let encoded = serde_json::to_string(self).unwrap_or_default().to_ascii_lowercase(); + [ + "oracle_path", + "oracle_bytes", + "needles", + "expected_file", + "expected_symbol", + "focused_command", + "argv", + "bundle_path", + "bundle_location", + "evaluator_answer", + ] + .iter() + .all(|forbidden| !encoded.contains(forbidden)) + } +} + +pub fn build_launch_plan( + manifest: &FrozenCorpusManifest, + schedule: &CorpusSchedule, + power_plan: &PowerPlan, + schedule_digest: &str, + power_plan_digest: &str, +) -> Result, Vec> { + let errors = schedule.validate(manifest, power_plan, power_plan_digest); + if !errors.is_empty() { + return Err(errors); + } + let manifest_digest = corpus_digest(manifest); + schedule + .entries + .iter() + .map(|entry| { + let task = manifest + .tasks + .iter() + .find(|task| task.id == entry.task_id) + .ok_or_else(|| "schedule references an unknown task".to_owned())?; + ArmLaunch::from_task_with_digests( + task, + entry.arm, + entry.repetition, + &manifest_digest, + schedule_digest, + power_plan_digest, + ) + }) + .collect::, _>>() + .map_err(|error| vec![error]) +} + +fn validate_entries_in_manifest_order( + schedule: &CorpusSchedule, + manifest: &FrozenCorpusManifest, + power_plan: &PowerPlan, + errors: &mut Vec, +) { + let mut offset = 0usize; + let mut holdout_pairs = BTreeMap::>::new(); + let calibration_arms = [FinalArm::FrontierDirect, FinalArm::NeedleMiss]; + for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Calibration) { + for arm in calibration_arms { + let Some(entry) = schedule.entries.get(offset) else { + errors.push("schedule is missing a calibration entry".to_owned()); + return; + }; + if entry.task_id != task.id + || entry.route != task.route + || entry.split != task.split + || entry.arm != arm + || entry.repetition != 0 + { + errors.push( + "schedule calibration entries differ from canonical task order".to_owned(), + ); + } + offset = offset.saturating_add(1); + } + } + for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Holdout) { + let mut task_pairs = BTreeSet::new(); + let mut pair_counts = BTreeMap::<(String, u32), usize>::new(); + let task_start = offset; + while let Some(entry) = schedule.entries.get(offset) { + if entry.task_id != task.id { + break; + } + if entry.route != task.route || entry.split != task.split { + errors.push("schedule holdout metadata differs from manifest task".to_owned()); + } + let pair = (entry.task_id.clone(), entry.repetition); + task_pairs.insert(pair.clone()); + *pair_counts.entry(pair.clone()).or_default() += 1; + holdout_pairs.entry(task.route).or_default().insert(pair); + let within_pair = (offset - task_start) % FinalArm::ALL.len(); + if entry.arm != FinalArm::ALL[within_pair] { + errors.push("schedule holdout arms are not in canonical FinalArm order".to_owned()); + } + let canonical_repetition = ((offset - task_start) / FinalArm::ALL.len()) as u32; + if entry.repetition != canonical_repetition { + errors.push( + "schedule holdout repetitions are not in canonical block order".to_owned(), + ); + } + offset = offset.saturating_add(1); + } + if task_pairs.is_empty() { + errors.push("holdout task is missing from schedule".to_owned()); + continue; + } + let repetitions = + task_pairs.iter().map(|(_, repetition)| *repetition).collect::>(); + if repetitions + .iter() + .copied() + .enumerate() + .any(|(index, repetition)| repetition != index as u32) + { + errors.push("holdout task repetitions are not contiguous".to_owned()); + } + if pair_counts.values().any(|count| *count != FinalArm::ALL.len()) { + errors.push("holdout task must contain one complete arm set per pair".to_owned()); + } + } + if offset != schedule.entries.len() { + errors.push("schedule contains duplicate or extra entries".to_owned()); + } + for route in [BenchmarkRoute::LocateImplementation, BenchmarkRoute::TraceStateFlow] { + let observed = holdout_pairs.get(&route).map_or(0, BTreeSet::len) as u32; + if power_plan.required_pairs(route) != Some(observed) { + errors.push(format!("schedule holdout pair count differs for route {route:?}")); + } + } +} + +pub fn digest_json(value: &T) -> String { + format!("b3:{}", blake3::hash(&serde_json::to_vec(value).unwrap_or_default()).to_hex()) +} + +pub fn raw_digest(bytes: &[u8]) -> String { + format!("b3:{}", blake3::hash(bytes).to_hex()) +} + +fn valid_digest(value: &str) -> bool { + value.len() == 67 + && value.starts_with("b3:") + && value[3..].bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::FocusedTestPolicyRef; + + fn task(split: CorpusSplit, route: BenchmarkRoute, id: &str) -> CorpusTask { + CorpusTask { + id: id.to_owned(), + route, + split, + repository_url: "https://example.invalid/repo.git".to_owned(), + repository_sha: "0123456789abcdef0123456789abcdef01234567".to_owned(), + prompt: "A bounded synthetic benchmark prompt with no answer material.".to_owned(), + material_class: CorpusMaterialClass::Synthetic, + focused_test_policy: FocusedTestPolicyRef { + identity: "synthetic-policy".to_owned(), + commitment: "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + .to_owned(), + }, + oracle_schema: "needle.sealed-oracle/1".to_owned(), + oracle_digest: "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + .to_owned(), + oracle_path: String::new(), + test_identifier: String::new(), + focused_command: Vec::new(), + } + } + + fn manifest() -> FrozenCorpusManifest { + let tasks = vec![ + task(CorpusSplit::Calibration, BenchmarkRoute::LocateImplementation, "cal-locate"), + task(CorpusSplit::Calibration, BenchmarkRoute::TraceStateFlow, "cal-trace"), + task(CorpusSplit::Holdout, BenchmarkRoute::LocateImplementation, "hold-locate"), + task(CorpusSplit::Holdout, BenchmarkRoute::TraceStateFlow, "hold-trace"), + ]; + FrozenCorpusManifest { + schema: "needle.frozen-corpus/4".to_owned(), + frozen_unix_ms: 1, + arms: FinalArm::ALL.to_vec(), + cost_model_path: "cost-model.json".to_owned(), + cost_model_digest: + "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), + next_pilot_path: "minimal-live-pilot.json".to_owned(), + next_pilot_digest: + "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), + campaign_path: Some("campaign.json".to_owned()), + campaign_digest: Some( + "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), + ), + schedule_path: None, + schedule_digest: None, + power_plan_path: None, + power_plan_digest: None, + sealed_bundle_schema: None, + sealed_bundle_digest: None, + tasks, + } + } + + fn plan(manifest: &FrozenCorpusManifest) -> PowerPlan { + PowerPlan { + schema: POWER_PLAN_SCHEMA.to_owned(), + plan_id: "synthetic".to_owned(), + manifest_digest: corpus_digest(manifest), + campaign_digest: manifest.campaign_digest.clone().unwrap(), + calibration_input_digest: + "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), + estimator_revision: "issue-7-structural-v1".to_owned(), + alpha_basis_points: 500, + target_power_basis_points: 9_000, + routes: vec![ + PowerRoutePlan { + route: BenchmarkRoute::LocateImplementation, + baseline_arm: FinalArm::FrontierDirect, + treatment_arm: FinalArm::NeedleMiss, + pair_key: "task_id:repetition".to_owned(), + observed_log_ratio_mean: -0.4, + observed_log_ratio_stddev: 0.2, + required_pairs: 1, + }, + PowerRoutePlan { + route: BenchmarkRoute::TraceStateFlow, + baseline_arm: FinalArm::FrontierDirect, + treatment_arm: FinalArm::NeedleMiss, + pair_key: "task_id:repetition".to_owned(), + observed_log_ratio_mean: -0.4, + observed_log_ratio_stddev: 0.2, + required_pairs: 1, + }, + ], + validated: true, + synthetic: true, + } + } + + fn schedule(manifest: &FrozenCorpusManifest, plan: &PowerPlan) -> CorpusSchedule { + let mut entries = Vec::new(); + for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Calibration) { + for arm in [FinalArm::FrontierDirect, FinalArm::NeedleMiss] { + entries.push(ScheduleEntry { + task_id: task.id.clone(), + route: task.route, + split: task.split, + arm, + repetition: 0, + }); + } + } + for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Holdout) { + for arm in FinalArm::ALL { + entries.push(ScheduleEntry { + task_id: task.id.clone(), + route: task.route, + split: task.split, + arm, + repetition: 0, + }); + } + } + let bytes = serde_json::to_vec(&plan).unwrap(); + CorpusSchedule { + schema: CORPUS_SCHEDULE_SCHEMA.to_owned(), + manifest_digest: corpus_digest(manifest), + power_plan_digest: raw_digest(&bytes), + automatic_retries: false, + entries, + } + } + + #[test] + fn schedule_is_canonical_and_launch_projection_binds_artifact_digests() { + let manifest = manifest(); + let plan = plan(&manifest); + let plan_bytes = serde_json::to_vec(&plan).unwrap(); + let plan_digest = raw_digest(&plan_bytes); + let schedule = schedule(&manifest, &plan); + let schedule_bytes = serde_json::to_vec(&schedule).unwrap(); + let schedule_digest = raw_digest(&schedule_bytes); + assert!(plan.validate(&manifest).is_empty()); + assert!(schedule.validate(&manifest, &plan, &plan_digest).is_empty()); + let launch = + build_launch_plan(&manifest, &schedule, &plan, &schedule_digest, &plan_digest).unwrap(); + assert_eq!(launch.len(), schedule.entries.len()); + assert_eq!(launch[0].manifest_digest, corpus_digest(&manifest)); + assert!(launch.iter().all(ArmLaunch::serialization_is_bounded)); + } + + #[test] + fn schedule_rejects_duplicate_reordered_and_syntactically_valid_stale_plan_digest() { + let manifest = manifest(); + let plan = plan(&manifest); + let plan_digest = raw_digest(&serde_json::to_vec(&plan).unwrap()); + let mut schedule = schedule(&manifest, &plan); + schedule.entries.reverse(); + schedule.entries.push(schedule.entries[0].clone()); + let errors = schedule.validate(&manifest, &plan, &plan_digest); + assert!(errors.iter().any(|error| error.contains("canonical") + || error.contains("duplicate") + || error.contains("extra"))); + let stale = "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + assert!( + schedule + .validate(&manifest, &plan, stale) + .iter() + .any(|error| error.contains("raw power-plan")) + ); + } + + #[test] + fn schedule_rejects_complete_holdout_blocks_in_reversed_repetition_order() { + let manifest = manifest(); + let mut plan = plan(&manifest); + for route in &mut plan.routes { + route.required_pairs = 2; + } + let canonical = schedule(&manifest, &plan); + let mut entries = canonical.entries[..4].to_vec(); + for task_id in ["hold-locate", "hold-trace"] { + let block = canonical + .entries + .iter() + .filter(|entry| entry.task_id == task_id) + .cloned() + .collect::>(); + let mut repetition_one = block.clone(); + repetition_one.iter_mut().for_each(|entry| entry.repetition = 1); + let mut repetition_zero = block; + repetition_zero.iter_mut().for_each(|entry| entry.repetition = 0); + entries.extend(repetition_one); + entries.extend(repetition_zero); + } + let mut reversed = canonical; + reversed.entries = entries; + let plan_digest = raw_digest(&serde_json::to_vec(&plan).unwrap()); + let errors = reversed.validate(&manifest, &plan, &plan_digest); + assert!(errors.iter().any(|error| error.contains("repetition") && error.contains("block"))); + } + + #[test] + fn public_manifest_and_launch_shapes_are_answer_free() { + let encoded = serde_json::to_string(&manifest()).unwrap(); + for prohibited in [ + "oracle_path", + "focused_command", + "test_identifier", + "needles", + "expected_symbol", + "bundle_location", + ] { + assert!(!encoded.contains(prohibited), "manifest leaked `{prohibited}`"); + } + let manifest = manifest(); + let plan = plan(&manifest); + let plan_bytes = serde_json::to_vec(&plan).unwrap(); + let plan_digest = raw_digest(&plan_bytes); + let schedule = schedule(&manifest, &plan); + let schedule_digest = raw_digest(&serde_json::to_vec(&schedule).unwrap()); + let launch = build_launch_plan(&manifest, &schedule, &plan, &schedule_digest, &plan_digest) + .unwrap() + .remove(0); + assert!(launch.serialization_is_bounded()); + } + + #[test] + fn unknown_schedule_fields_and_synthetic_plan_are_fail_closed() { + let malformed = r#"{"schema":"needle.corpus-schedule/1","manifest_digest":"b3:0000000000000000000000000000000000000000000000000000000000000000","power_plan_digest":"b3:0000000000000000000000000000000000000000000000000000000000000000","automatic_retries":false,"entries":[],"answer":"leak"}"#; + assert!(serde_json::from_str::(malformed).is_err()); + let manifest = manifest(); + let plan = plan(&manifest); + assert!(plan.synthetic); + } +} diff --git a/crates/needle-bench/src/sealed_oracle.rs b/crates/needle-bench/src/sealed_oracle.rs new file mode 100644 index 0000000..46178ab --- /dev/null +++ b/crates/needle-bench/src/sealed_oracle.rs @@ -0,0 +1,767 @@ +use crate::{ + CorpusMaterialClass, FocusedTestPolicyRef, FrozenCorpusManifest, QualityOracleResult, + QualityOracleSpec, read_bounded_file, +}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::path::{Component, Path}; + +pub const SEALED_BUNDLE_INDEX_SCHEMA: &str = "needle.sealed-oracle-index/1"; +pub const SEALED_ORACLE_SCHEMA: &str = "needle.sealed-oracle/1"; +pub const MAX_SEALED_INDEX_BYTES: usize = 512 * 1024; +pub const MAX_SEALED_DOCUMENT_BYTES: usize = 512 * 1024; +pub const MAX_SEALED_ENTRIES: usize = 512; +const MAX_IDENTIFIER_BYTES: usize = 256; +const MAX_PATH_BYTES: usize = 512; +const MAX_POLICY_STRING_BYTES: usize = 4_096; +const MAX_POLICY_VECTOR_ITEMS: usize = 64; +const MAX_POLICY_ARG_BYTES: usize = 256; + +/// Evaluator-only index entry. `relative_path` is accepted only at the +/// evaluator boundary and never appears in a public manifest or ArmLaunch. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SealedOracleIndexEntry { + pub task_id: String, + pub material_class: CorpusMaterialClass, + pub repository_sha: String, + pub oracle_schema: String, + pub oracle_digest: String, + pub focused_test_policy: FocusedTestPolicyRef, + pub relative_path: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SealedOracleIndex { + pub schema: String, + pub entries: Vec, +} + +/// This document is intentionally answer-bearing and is only deserialized by +/// the evaluator process. It is never a field on a corpus task or launch DTO. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SealedOracleDocument { + pub schema: String, + pub task_id: String, + pub repository_sha: String, + pub focused_test_policy: FocusedTestPolicyRef, + pub focused_test: SealedFocusedTestPolicy, + pub quality: SealedQualityPolicy, +} + +/// Evaluator-only focused-test contract. The argv is sealed and never +/// projected into a public manifest or launch DTO; only this exact direct +/// offline Cargo integration-test shape is accepted. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SealedFocusedTestPolicy { + pub identifier: String, + pub argv: Vec, +} + +impl SealedFocusedTestPolicy { + pub fn is_well_formed(&self) -> bool { + let expected = [ + "cargo", + "test", + "--offline", + "--test", + "integration", + self.identifier.as_str(), + "--", + "--exact", + ]; + !self.identifier.trim().is_empty() + && self.identifier.len() <= MAX_POLICY_ARG_BYTES + && !self.identifier.chars().any(char::is_whitespace) + && self.argv.len() == expected.len() + && self + .argv + .iter() + .all(|argument| !argument.is_empty() && argument.len() <= MAX_POLICY_ARG_BYTES) + && self.argv.iter().map(String::as_str).eq(expected) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SealedQualityPolicy { + pub required_files: Vec, + pub required_symbols: Vec, + pub required_claims: Vec, + pub forbidden_claims: Vec, + pub focused_test_command: String, + pub accepted_focused_test_identifiers: Vec, + pub focused_test_required: bool, +} + +impl SealedQualityPolicy { + pub fn is_well_formed(&self) -> bool { + self.required_files.len() <= MAX_POLICY_VECTOR_ITEMS + && self.required_symbols.len() <= MAX_POLICY_VECTOR_ITEMS + && self.required_claims.len() <= MAX_POLICY_VECTOR_ITEMS + && self.forbidden_claims.len() <= MAX_POLICY_VECTOR_ITEMS + && self.accepted_focused_test_identifiers.len() <= MAX_POLICY_VECTOR_ITEMS + && self + .required_files + .iter() + .chain(&self.required_symbols) + .chain(&self.required_claims) + .chain(&self.forbidden_claims) + .chain(&self.accepted_focused_test_identifiers) + .all(|value| value.len() <= MAX_POLICY_STRING_BYTES) + && self.focused_test_command.len() <= MAX_POLICY_STRING_BYTES + && self + .required_files + .iter() + .chain(&self.required_symbols) + .chain(&self.required_claims) + .chain(&self.forbidden_claims) + .chain(&self.accepted_focused_test_identifiers) + .all(|value| !value.trim().is_empty()) + && !self.focused_test_command.trim().is_empty() + } + + pub fn as_quality_oracle_spec(&self) -> QualityOracleSpec { + QualityOracleSpec { + required_files: self.required_files.clone(), + required_symbols: self.required_symbols.clone(), + required_claims: self.required_claims.clone(), + forbidden_claims: self.forbidden_claims.clone(), + focused_test_command: self.focused_test_command.clone(), + accepted_focused_test_identifiers: self.accepted_focused_test_identifiers.clone(), + focused_test_required: self.focused_test_required, + } + } +} + +#[derive(Serialize)] +struct FocusedTestPolicyMaterial<'a> { + identity: &'a str, + focused_test: &'a SealedFocusedTestPolicy, + quality: &'a SealedQualityPolicy, +} + +/// Commitment used by public task refs and evaluator-owned documents. It is +/// computed from the complete bounded structured policy, never from an answer +/// string or filesystem location. +pub fn focused_test_policy_commitment( + identity: &str, + focused_test: &SealedFocusedTestPolicy, + quality: &SealedQualityPolicy, +) -> String { + let material = FocusedTestPolicyMaterial { identity, focused_test, quality }; + format!("b3:{}", blake3::hash(&serde_json::to_vec(&material).unwrap_or_default()).to_hex()) +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct BundleTaskValidation { + pub task_id: String, + pub bound: bool, + pub errors: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct SealedBundleValidationReport { + pub schema: String, + pub index_digest: Option, + pub bundle_root: Option, + pub production_material: bool, + pub production_bundle_ready: bool, + pub tasks: Vec, + pub errors: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SealedEvaluationResult { + pub schema: String, + pub task_id: String, + pub response_digest: String, + pub oracle_digest: Option, + pub quality_passed: Option, + pub quality_failures: Vec, + pub focused_test_evidence_status: String, + pub infrastructure_errors: Vec, + pub evaluator_errors: Vec, + pub error_codes: Vec, +} + +impl SealedBundleValidationReport { + fn unavailable(reason: impl Into) -> Self { + Self { + schema: "needle.sealed-oracle-validation/1".to_owned(), + index_digest: None, + bundle_root: None, + production_material: false, + production_bundle_ready: false, + tasks: Vec::new(), + errors: vec![reason.into()], + } + } +} + +/// Validate a sealed evaluator index and its bytes under a caller-provided +/// private bundle root. Diagnostics are bounded so malformed material cannot +/// flood preflight output. +pub fn validate_sealed_bundle( + manifest: &FrozenCorpusManifest, + index_path: Option<&Path>, + bundle_root: Option<&Path>, +) -> SealedBundleValidationReport { + const MAX_ERRORS: usize = 24; + let (Some(index_path), Some(bundle_root)) = (index_path, bundle_root) else { + return SealedBundleValidationReport::unavailable( + "production sealed evaluator bundle is unavailable", + ); + }; + let index_bytes = match read_bounded_file(index_path, MAX_SEALED_INDEX_BYTES) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::InvalidData => { + return SealedBundleValidationReport::unavailable( + "sealed evaluator index exceeds bounded byte limit", + ); + } + Err(_) => { + return SealedBundleValidationReport::unavailable( + "sealed evaluator index cannot be read", + ); + } + }; + let index_digest = format!("b3:{}", blake3::hash(&index_bytes).to_hex()); + if let Some(expected) = manifest.sealed_bundle_digest.as_deref() + && expected != index_digest + { + return SealedBundleValidationReport { + index_digest: Some(index_digest), + bundle_root: None, + ..SealedBundleValidationReport::unavailable( + "sealed evaluator index digest differs from manifest commitment", + ) + }; + } + let index: SealedOracleIndex = match serde_json::from_slice(&index_bytes) { + Ok(index) => index, + Err(_) => { + return SealedBundleValidationReport { + index_digest: Some(index_digest), + bundle_root: None, + ..SealedBundleValidationReport::unavailable( + "sealed evaluator index JSON is invalid", + ) + }; + } + }; + let root = match bundle_root.canonicalize() { + Ok(root) => root, + Err(_) => { + return SealedBundleValidationReport { + index_digest: Some(index_digest), + bundle_root: None, + ..SealedBundleValidationReport::unavailable( + "sealed evaluator bundle root is unavailable", + ) + }; + } + }; + let mut errors = Vec::new(); + if index.schema != SEALED_BUNDLE_INDEX_SCHEMA { + errors.push("sealed evaluator index schema is unsupported".to_owned()); + } + if index.entries.is_empty() || index.entries.len() > MAX_SEALED_ENTRIES { + errors.push("sealed evaluator index entry count is out of bounds".to_owned()); + } + let mut seen = BTreeSet::new(); + let mut task_reports = Vec::new(); + let mut all_bound = index.schema == SEALED_BUNDLE_INDEX_SCHEMA + && !index.entries.is_empty() + && index.entries.len() <= MAX_SEALED_ENTRIES; + for entry in &index.entries { + if entry.task_id.len() > MAX_IDENTIFIER_BYTES + || entry.repository_sha.len() > MAX_IDENTIFIER_BYTES + || entry.oracle_schema.len() > MAX_IDENTIFIER_BYTES + || entry.oracle_digest.len() > MAX_IDENTIFIER_BYTES + || entry.focused_test_policy.identity.len() > MAX_IDENTIFIER_BYTES + || entry.focused_test_policy.commitment.len() > MAX_IDENTIFIER_BYTES + || entry.relative_path.len() > MAX_PATH_BYTES + { + push_error( + &mut errors, + MAX_ERRORS, + "sealed entry exceeds bounded field length".to_owned(), + ); + all_bound = false; + continue; + } + if !seen.insert(entry.task_id.as_str()) { + all_bound = false; + push_error( + &mut errors, + MAX_ERRORS, + format!("duplicate sealed task `{}`", entry.task_id), + ); + continue; + } + let Some(task) = manifest.tasks.iter().find(|task| task.id == entry.task_id) else { + all_bound = false; + push_error( + &mut errors, + MAX_ERRORS, + format!("sealed task `{}` is not in manifest", entry.task_id), + ); + continue; + }; + let mut task_errors = Vec::new(); + if (entry.material_class != CorpusMaterialClass::ProductionSealed + || task.material_class != CorpusMaterialClass::ProductionSealed) + && entry.material_class != task.material_class + { + task_errors.push("material class differs".to_owned()); + } + if entry.repository_sha != task.repository_sha { + task_errors.push("repository SHA differs".to_owned()); + } + if entry.oracle_schema != task.oracle_schema || entry.oracle_digest != task.oracle_digest { + task_errors.push("oracle schema or digest differs".to_owned()); + } + if entry.focused_test_policy != task.focused_test_policy { + task_errors.push("focused-test policy commitment differs".to_owned()); + } + let path = Path::new(&entry.relative_path); + if !safe_relative_path(path) { + task_errors.push("sealed oracle path is not safe and relative".to_owned()); + } + let bytes = if task_errors.is_empty() { + let candidate = root.join(path); + match candidate.canonicalize() { + Ok(canonical) if canonical.starts_with(&root) => { + match read_bounded_file(&canonical, MAX_SEALED_DOCUMENT_BYTES) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::InvalidData => { + task_errors.push( + "sealed oracle document exceeds bounded byte limit".to_owned(), + ); + None + } + Err(_) => { + task_errors.push("sealed oracle bytes cannot be read".to_owned()); + None + } + } + } + Ok(_) => { + task_errors.push("sealed oracle path escapes bundle root".to_owned()); + None + } + Err(_) => { + task_errors.push("sealed oracle path is unavailable".to_owned()); + None + } + } + } else { + None + }; + if let Some(bytes) = bytes { + let actual = format!("b3:{}", blake3::hash(&bytes).to_hex()); + if actual != entry.oracle_digest { + task_errors.push("sealed oracle digest differs".to_owned()); + } + match serde_json::from_slice::(&bytes) { + Ok(document) => { + if document.schema != SEALED_ORACLE_SCHEMA + || document.task_id != task.id + || document.repository_sha != task.repository_sha + || document.focused_test_policy != task.focused_test_policy + || !document.focused_test.is_well_formed() + || document.quality.focused_test_command != document.focused_test.identifier + || !document.quality.is_well_formed() + || focused_test_policy_commitment( + &document.focused_test_policy.identity, + &document.focused_test, + &document.quality, + ) != task.focused_test_policy.commitment + { + task_errors.push("sealed oracle metadata does not bind to task".to_owned()); + } + } + Err(_) => task_errors.push("sealed oracle JSON is invalid".to_owned()), + } + } + let bound = task_errors.is_empty(); + all_bound &= bound; + for error in &task_errors { + push_error(&mut errors, MAX_ERRORS, format!("{}: {error}", entry.task_id)); + } + task_reports.push(BundleTaskValidation { + task_id: entry.task_id.clone(), + bound, + errors: task_errors, + }); + } + let expected = manifest.tasks.iter().map(|task| task.id.as_str()).collect::>(); + if seen != expected { + push_error( + &mut errors, + MAX_ERRORS, + "sealed bundle task set differs from manifest".to_owned(), + ); + all_bound = false; + } + let production_material = all_bound + && !manifest.tasks.is_empty() + && manifest + .tasks + .iter() + .all(|task| task.material_class == CorpusMaterialClass::ProductionSealed); + SealedBundleValidationReport { + schema: "needle.sealed-oracle-validation/1".to_owned(), + index_digest: Some(index_digest), + bundle_root: None, + production_material, + production_bundle_ready: production_material && errors.is_empty(), + tasks: task_reports, + errors, + } +} + +fn safe_relative_path(path: &Path) -> bool { + path.is_relative() + && !path.as_os_str().is_empty() + && path.components().all(|component| matches!(component, Component::Normal(_))) +} + +fn push_error(errors: &mut Vec, limit: usize, error: String) { + if errors.len() < limit { + errors.push(error); + } +} + +/// Keep a direct helper for callers that need to commit an index's raw bytes. +pub fn sealed_index_digest(bytes: &[u8]) -> String { + format!("b3:{}", blake3::hash(bytes).to_hex()) +} + +/// Evaluator-only response boundary. The caller supplies the private bundle +/// and completed response; the returned projection contains only digests, +/// quality booleans/failure codes, and bounded infrastructure diagnostics. +pub fn evaluate_sealed_response( + manifest: &FrozenCorpusManifest, + index_path: &Path, + bundle_root: &Path, + task_id: &str, + response: &str, + focused_test_passed: Option, +) -> SealedEvaluationResult { + const MAX_ERRORS: usize = 24; + let response_digest = format!("b3:{}", blake3::hash(response.as_bytes()).to_hex()); + let mut result = SealedEvaluationResult { + schema: "needle.sealed-evaluation/1".to_owned(), + task_id: task_id.chars().take(MAX_IDENTIFIER_BYTES).collect(), + response_digest, + oracle_digest: manifest + .tasks + .iter() + .find(|task| task.id == task_id) + .map(|task| task.oracle_digest.clone()), + quality_passed: None, + quality_failures: Vec::new(), + focused_test_evidence_status: match focused_test_passed { + Some(true) => "passed".to_owned(), + Some(false) => "failed".to_owned(), + None => "not_run".to_owned(), + }, + infrastructure_errors: Vec::new(), + evaluator_errors: Vec::new(), + error_codes: Vec::new(), + }; + if task_id.is_empty() + || task_id.len() > MAX_IDENTIFIER_BYTES + || response.len() > MAX_SEALED_DOCUMENT_BYTES + { + result.evaluator_errors.push("evaluation input exceeds bounded limits".to_owned()); + result.error_codes.push("input_bounds".to_owned()); + return result; + } + let bundle = validate_sealed_bundle(manifest, Some(index_path), Some(bundle_root)); + if !bundle.errors.is_empty() { + result.infrastructure_errors.push("sealed bundle validation failed".to_owned()); + result.error_codes.push("bundle_invalid".to_owned()); + return result; + } + let index_bytes = match read_bounded_file(index_path, MAX_SEALED_INDEX_BYTES) { + Ok(bytes) => bytes, + _ => { + result.infrastructure_errors.push("sealed evaluator index cannot be read".to_owned()); + result.error_codes.push("index_unavailable".to_owned()); + return result; + } + }; + let index = match serde_json::from_slice::(&index_bytes) { + Ok(index) => index, + Err(_) => { + result.infrastructure_errors.push("sealed evaluator index is invalid".to_owned()); + result.error_codes.push("index_invalid".to_owned()); + return result; + } + }; + let Some(entry) = index.entries.iter().find(|entry| entry.task_id == task_id) else { + result.evaluator_errors.push("evaluation task is absent from sealed index".to_owned()); + result.error_codes.push("task_missing".to_owned()); + return result; + }; + let root = match bundle_root.canonicalize() { + Ok(root) => root, + Err(_) => { + result.infrastructure_errors.push("sealed bundle root is unavailable".to_owned()); + result.error_codes.push("bundle_unavailable".to_owned()); + return result; + } + }; + let Some(path) = root.join(&entry.relative_path).canonicalize().ok() else { + result.infrastructure_errors.push("sealed oracle document is unavailable".to_owned()); + result.error_codes.push("oracle_unavailable".to_owned()); + return result; + }; + let candidate = root.join(&entry.relative_path); + if !path.starts_with(&root) || !candidate.exists() { + result.infrastructure_errors.push("sealed oracle path is outside bundle".to_owned()); + result.error_codes.push("oracle_path".to_owned()); + return result; + } + let document = match read_bounded_file(&path, MAX_SEALED_DOCUMENT_BYTES) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + { + Some(document) => document, + None => { + result.infrastructure_errors.push("sealed oracle document is invalid".to_owned()); + result.error_codes.push("oracle_invalid".to_owned()); + return result; + } + }; + let quality = QualityOracleResult::evaluate( + &document.quality.as_quality_oracle_spec(), + response, + focused_test_passed, + ); + result.quality_passed = Some(quality.passed); + result.quality_failures = quality.failures.into_iter().take(MAX_ERRORS).collect(); + if !quality.passed { + result.error_codes.push("quality_failed".to_owned()); + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CorpusTask; + use std::fs; + + #[test] + fn missing_bundle_is_fail_closed_and_bounded() { + let report = validate_sealed_bundle( + &FrozenCorpusManifest { + schema: "needle.frozen-corpus/4".to_owned(), + frozen_unix_ms: 1, + arms: Vec::new(), + cost_model_path: String::new(), + cost_model_digest: String::new(), + next_pilot_path: String::new(), + next_pilot_digest: String::new(), + campaign_path: None, + campaign_digest: None, + schedule_path: None, + schedule_digest: None, + power_plan_path: None, + power_plan_digest: None, + sealed_bundle_schema: None, + sealed_bundle_digest: None, + tasks: Vec::new(), + }, + None, + None, + ); + assert!(!report.production_bundle_ready); + assert!(report.bundle_root.is_none()); + assert!(report.errors.len() <= 24); + } + + #[test] + fn unsafe_and_absolute_bundle_paths_fail_closed() { + assert!(!safe_relative_path(Path::new("../oracle.json"))); + assert!(!safe_relative_path(Path::new("/tmp/oracle.json"))); + assert!(!safe_relative_path(Path::new(""))); + assert!(safe_relative_path(Path::new("task/oracle.json"))); + } + + #[test] + fn malformed_focused_test_command_is_rejected() { + let malformed = SealedFocusedTestPolicy { + identifier: "suite::case".to_owned(), + argv: ["sh", "-c", "cargo test suite::case"].into_iter().map(str::to_owned).collect(), + }; + assert!(!malformed.is_well_formed()); + } + + #[test] + fn bundle_binding_rejects_mutation_swap_and_unknown_fields() { + let root = std::env::temp_dir().join(format!("needle-sealed-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).unwrap(); + let quality = SealedQualityPolicy { + required_files: Vec::new(), + required_symbols: Vec::new(), + required_claims: Vec::new(), + forbidden_claims: Vec::new(), + focused_test_command: "synthetic-test".to_owned(), + accepted_focused_test_identifiers: Vec::new(), + focused_test_required: false, + }; + let focused_test = SealedFocusedTestPolicy { + identifier: "synthetic-test".to_owned(), + argv: [ + "cargo", + "test", + "--offline", + "--test", + "integration", + "synthetic-test", + "--", + "--exact", + ] + .into_iter() + .map(str::to_owned) + .collect(), + }; + let policy = FocusedTestPolicyRef { + identity: "synthetic-policy".to_owned(), + commitment: focused_test_policy_commitment("synthetic-policy", &focused_test, &quality), + }; + let document = |task_id: &str| SealedOracleDocument { + schema: SEALED_ORACLE_SCHEMA.to_owned(), + task_id: task_id.to_owned(), + repository_sha: "0123456789abcdef0123456789abcdef01234567".to_owned(), + focused_test_policy: policy.clone(), + focused_test: focused_test.clone(), + quality: quality.clone(), + }; + let first = serde_json::to_vec(&document("one")).unwrap(); + let second = serde_json::to_vec(&document("two")).unwrap(); + fs::write(root.join("one.json"), &first).unwrap(); + fs::write(root.join("two.json"), &second).unwrap(); + let task = |id: &str, path: &str, bytes: &[u8]| CorpusTask { + id: id.to_owned(), + route: crate::BenchmarkRoute::LocateImplementation, + split: crate::CorpusSplit::Calibration, + repository_url: "https://example.invalid/repo.git".to_owned(), + repository_sha: "0123456789abcdef0123456789abcdef01234567".to_owned(), + prompt: "A bounded synthetic prompt with no answer-bearing material.".to_owned(), + material_class: CorpusMaterialClass::ProductionSealed, + focused_test_policy: policy.clone(), + oracle_schema: SEALED_ORACLE_SCHEMA.to_owned(), + oracle_digest: sealed_index_digest(bytes), + oracle_path: path.to_owned(), + test_identifier: String::new(), + focused_command: Vec::new(), + }; + let manifest = FrozenCorpusManifest { + schema: "needle.frozen-corpus/4".to_owned(), + frozen_unix_ms: 1, + arms: Vec::new(), + cost_model_path: String::new(), + cost_model_digest: String::new(), + next_pilot_path: String::new(), + next_pilot_digest: String::new(), + campaign_path: None, + campaign_digest: None, + schedule_path: None, + schedule_digest: None, + power_plan_path: None, + power_plan_digest: None, + sealed_bundle_schema: None, + sealed_bundle_digest: None, + tasks: vec![task("one", "one.json", &first), task("two", "two.json", &second)], + }; + let index = SealedOracleIndex { + schema: SEALED_BUNDLE_INDEX_SCHEMA.to_owned(), + entries: vec![ + SealedOracleIndexEntry { + task_id: "one".to_owned(), + material_class: CorpusMaterialClass::ProductionSealed, + repository_sha: manifest.tasks[0].repository_sha.clone(), + oracle_schema: SEALED_ORACLE_SCHEMA.to_owned(), + oracle_digest: manifest.tasks[0].oracle_digest.clone(), + focused_test_policy: policy.clone(), + relative_path: "one.json".to_owned(), + }, + SealedOracleIndexEntry { + task_id: "two".to_owned(), + material_class: CorpusMaterialClass::ProductionSealed, + repository_sha: manifest.tasks[1].repository_sha.clone(), + oracle_schema: SEALED_ORACLE_SCHEMA.to_owned(), + oracle_digest: manifest.tasks[1].oracle_digest.clone(), + focused_test_policy: policy, + relative_path: "two.json".to_owned(), + }, + ], + }; + let index_path = root.join("index.json"); + fs::write(&index_path, serde_json::to_vec(&index).unwrap()).unwrap(); + let valid = validate_sealed_bundle(&manifest, Some(&index_path), Some(&root)); + assert!(valid.production_bundle_ready, "{:?}", valid.errors); + fs::copy(root.join("one.json"), root.join("two.json")).unwrap(); + let swapped = validate_sealed_bundle(&manifest, Some(&index_path), Some(&root)); + assert!(!swapped.production_bundle_ready); + assert!(swapped.errors.iter().any(|error| error.contains("two"))); + fs::write(root.join("two.json"), &second).unwrap(); + fs::remove_file(root.join("one.json")).unwrap(); + let missing = validate_sealed_bundle(&manifest, Some(&index_path), Some(&root)); + assert!(!missing.production_bundle_ready); + assert!(missing.errors.iter().any(|error| error.contains("one"))); + fs::write(root.join("one.json"), &first).unwrap(); + fs::write(root.join("two.json"), b"mutated").unwrap(); + let mutated = validate_sealed_bundle(&manifest, Some(&index_path), Some(&root)); + assert!(!mutated.production_bundle_ready); + assert!(mutated.errors.iter().any(|error| error.contains("two"))); + assert!( + serde_json::from_str::( + r#"{"schema":"needle.sealed-oracle-index/1","entries":[],"answer":"leak"}"# + ) + .is_err() + ); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn synthetic_fixture_evaluator_returns_only_bounded_projection() { + let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/corpus/router-cache"); + let manifest: FrozenCorpusManifest = + serde_json::from_slice(&fs::read(root.join("manifest.json")).expect("manifest")) + .expect("manifest JSON"); + let bundle_root = root.join("synthetic-sealed"); + let report = validate_sealed_bundle( + &manifest, + Some(&bundle_root.join("index.json")), + Some(&bundle_root), + ); + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert!(!report.production_bundle_ready); + let result = evaluate_sealed_response( + &manifest, + &bundle_root.join("index.json"), + &bundle_root, + "ripgrep-glob-case-insensitive-locate-calibration", + "The implementation is globs in crates/core/flags/hiargs.rs; the test is misc::glob_always_case_insensitive.", + Some(true), + ); + assert_eq!(result.quality_passed, Some(true)); + let encoded = serde_json::to_string(&result).unwrap(); + assert!(!encoded.contains("synthetic-sealed")); + assert!(!encoded.contains("hiargs.rs")); + assert!(result.error_codes.is_empty()); + } +} diff --git a/crates/needle-bench/tests/publication_surface.rs b/crates/needle-bench/tests/publication_surface.rs index 77e6614..c92bfc6 100644 --- a/crates/needle-bench/tests/publication_surface.rs +++ b/crates/needle-bench/tests/publication_surface.rs @@ -1,3 +1,7 @@ +use needle_bench::{ + ArmLaunch, CorpusSchedule, FrozenCorpusManifest, PowerPlan, SealedOracleIndex, + build_launch_plan, corpus_digest, raw_digest, validate_sealed_bundle, +}; use serde_json::Value; use std::collections::BTreeSet; use std::fs; @@ -146,3 +150,66 @@ fn assert_evidence_paths_exist(value: &Value, root: &Path) { _ => {} } } + +#[test] +fn public_v4_fixture_is_answer_free_and_exactly_committed() { + let root = workspace().join("benchmarks/corpus/router-cache"); + let manifest_bytes = fs::read(root.join("manifest.json")).expect("manifest"); + let manifest: FrozenCorpusManifest = + serde_json::from_slice(&manifest_bytes).expect("manifest JSON"); + assert_eq!(manifest.schema, "needle.frozen-corpus/4"); + assert!( + manifest + .tasks + .iter() + .all(|task| task.material_class == needle_bench::CorpusMaterialClass::Synthetic) + ); + assert!(manifest.tasks.iter().any(|task| task.id == "ripgrep-no-ignore-vcs-locate-holdout")); + assert!(manifest.tasks.iter().any(|task| task.id == "ripgrep-null-data-trace-holdout")); + let manifest_value = serde_json::to_value(&manifest).unwrap(); + let encoded = manifest_value.to_string(); + for prohibited in [ + "oracle_path", + "test_identifier", + "focused_command", + "needles", + "expected_symbol", + "bundle_location", + ] { + assert!(!encoded.contains(prohibited), "public manifest leaked {prohibited}"); + } + assert!( + manifest.schedule_digest.as_deref() + == Some(raw_digest(&fs::read(root.join("schedule.json")).unwrap()).as_str()) + ); + assert!( + manifest.power_plan_digest.as_deref() + == Some(raw_digest(&fs::read(root.join("power-plan.json")).unwrap()).as_str()) + ); + let schedule: CorpusSchedule = + serde_json::from_slice(&fs::read(root.join("schedule.json")).unwrap()).unwrap(); + let plan: PowerPlan = + serde_json::from_slice(&fs::read(root.join("power-plan.json")).unwrap()).unwrap(); + let plan_digest = raw_digest(&fs::read(root.join("power-plan.json")).unwrap()); + let schedule_digest = raw_digest(&fs::read(root.join("schedule.json")).unwrap()); + assert_eq!(plan.manifest_digest, corpus_digest(&manifest)); + assert!(plan.validate(&manifest).is_empty()); + assert!(schedule.validate(&manifest, &plan, &plan_digest).is_empty()); + let launches = + build_launch_plan(&manifest, &schedule, &plan, &schedule_digest, &plan_digest).unwrap(); + assert_eq!(launches.len(), schedule.entries.len()); + assert!(launches.iter().all(ArmLaunch::serialization_is_bounded)); + let bundle_root = root.join("synthetic-sealed"); + let bundle = validate_sealed_bundle( + &manifest, + Some(&bundle_root.join("index.json")), + Some(&bundle_root), + ); + assert!(bundle.errors.is_empty(), "{:?}", bundle.errors); + assert!(!bundle.production_bundle_ready); + assert!(!bundle.production_material); + let index: SealedOracleIndex = + serde_json::from_slice(&fs::read(bundle_root.join("index.json")).unwrap()).unwrap(); + assert_eq!(index.entries.len(), manifest.tasks.len()); + assert!(bundle.bundle_root.is_none()); +} diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 24ae2b9..30ca20d 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -41,6 +41,18 @@ Economic arms must share: - main model, reasoning, service tier, and pricing snapshot; - independent hidden quality oracle. +Public corpus manifests use `needle.frozen-corpus/4`: they contain only +answer-free prompts, source identities, policy commitments, and oracle +commitments. Answer bytes and quality/test policy live in an evaluator-owned +sealed bundle indexed outside the public manifest. The checked-in router-cache +tasks and power plan are synthetic fixtures and are permanently ineligible for +provider evidence. + +The app provider path remains fail-closed until an isolated executor/broker +consumes only the bounded ArmLaunch projection. Evaluator callers must keep +private sealed material unmounted and inaccessible to the runner identity; the +offline protocol does not prove filesystem ACLs or process isolation. + Wording may differ only when the compiled semantic need remains equivalent. Narrower, wider, residual, or mutation requests are separate cache-behavior experiments. @@ -147,9 +159,10 @@ No powered corpus or publishable general savings interval exists. Before any provider run: -1. freeze task, SHA, request, oracle, models, pricing, and retry policy; +1. freeze task, SHA, request, oracle commitment, models, pricing, and retry policy; 2. run deterministic simulator and current native preflight; -3. verify source integrity and cleanup; +3. verify the immutable schedule, validated PowerPlan, production sealed bundle, + source integrity, and cleanup; 4. produce a complete cost estimate; 5. obtain explicit human approval for that exact stage; 6. execute once unless the approved protocol explicitly permits repetition.