diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index 767f9d7..9aaf734 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -142,10 +142,13 @@ validation. - Linux and macOS lack live platform evidence. - The accepted economic results are calibrations, not a powered corpus or general savings claim. The public v4 benchmark manifest is answer-free and - synthetic; the offline protocol and synthetic evaluator fixture are complete, - but real private material, a non-synthetic validated PowerPlan, and a - provider run are absent. App provider execution remains fail-closed until an - isolated executor/broker consumes only ArmLaunch. This does not prove + synthetic. Deterministic power planning, exact schedule/observation binding, + fail-closed paired analysis, and the two-sided 95% BCa report path are + implemented and offline validated. The checked-in plan is reproducible from + explicitly synthetic calibration observations and remains permanently + claim-ineligible. Real private material, a non-synthetic validated PowerPlan, + and a provider run are absent. App provider execution remains fail-closed + until an isolated executor/broker consumes only ArmLaunch. This does not prove filesystem ACLs or process isolation; callers must keep private bundles unmounted and inaccessible to the runner identity. - No public beta, support channel, or compatibility window exists. diff --git a/benchmarks/README.md b/benchmarks/README.md index 535426c..68d6864 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -72,6 +72,10 @@ maintainers; no external bundle path is present in public manifests or arm launch projections. A real sealed bundle and validated production PowerPlan are absent from this checkout, so offline preflight remains fail-closed. The synthetic bundle is complete for evaluator tests but cannot enable a provider -run. A future caller must keep private evaluator material unmounted and -inaccessible to the runner identity; this protocol does not claim ACL or -process-isolation proof. +run. `router-cache/synthetic-calibration-observations.jsonl` deterministically +reproduces the checked-in synthetic PowerPlan, while +`router-cache/synthetic-final-observations.jsonl` exercises the exact paired +holdout contract and its fail-closed synthetic boundary. None of these +artifacts is economic evidence. A future caller must keep private evaluator +material unmounted and inaccessible to the runner identity; this protocol does +not claim ACL or process-isolation proof. diff --git a/benchmarks/corpus/router-cache/campaign.json b/benchmarks/corpus/router-cache/campaign.json index a48ca0a..f40ac42 100644 --- a/benchmarks/corpus/router-cache/campaign.json +++ b/benchmarks/corpus/router-cache/campaign.json @@ -1,6 +1,7 @@ { "schema": "needle.multi-task-campaign/2", - "schedule_digest": "b3:c853b4ed84d8e9119013d788a7c769c63fd77f63a947706fbe4c128b65ed51d5", + "schedule_digest": "b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd", + "task_ids": [], "paid_arms": [ "frontier_direct", "needle_miss" @@ -15,6 +16,7 @@ "native_subagent", "escalation" ], + "repetitions_per_task": 0, "automatic_retries": false, "statistical_claim": false, "bootstrap_resamples": 10000, diff --git a/benchmarks/corpus/router-cache/manifest.json b/benchmarks/corpus/router-cache/manifest.json index a92aecc..a75e995 100644 --- a/benchmarks/corpus/router-cache/manifest.json +++ b/benchmarks/corpus/router-cache/manifest.json @@ -16,11 +16,11 @@ "next_pilot_path": "minimal-live-pilot.json", "next_pilot_digest": "b3:18a434227eb9abf2381eb0a65348f0dba66301ccba7dbdeb57c7933b078180cb", "campaign_path": "campaign.json", - "campaign_digest": "b3:082239af1ebe785555801af74c817c5e49c143759cd3e62ef6607f54dadf5e1d", + "campaign_digest": "b3:a434abd47f654b222576e60c29f1640fd311d7fa81fa66419018fb55b008329b", "schedule_path": "schedule.json", - "schedule_digest": "b3:c853b4ed84d8e9119013d788a7c769c63fd77f63a947706fbe4c128b65ed51d5", + "schedule_digest": "b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd", "power_plan_path": "power-plan.json", - "power_plan_digest": "b3:8605f040792744907aa94d3ce017a211b774c16521dbe744acb7a1b587647630", + "power_plan_digest": "b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6", "sealed_bundle_schema": "needle.sealed-oracle-index/1", "sealed_bundle_digest": "b3:3750e933fa1226fed1892dc4ba3ddad4f84fbf6bd58135d8f26355103dc681c7", "tasks": [ diff --git a/benchmarks/corpus/router-cache/power-plan.json b/benchmarks/corpus/router-cache/power-plan.json index d7367fc..dd41f73 100644 --- a/benchmarks/corpus/router-cache/power-plan.json +++ b/benchmarks/corpus/router-cache/power-plan.json @@ -1,10 +1,10 @@ { - "schema": "needle.power-plan/1", - "plan_id": "synthetic-router-cache-v1", + "schema": "needle.power-plan/2", + "plan_id": "power-plan-1d6bfc9443debec0", "manifest_digest": "b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a", - "campaign_digest": "b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864", - "calibration_input_digest": "b3:4d27a8427bd3655d59e88c37efadcccfca4a7d82cd4a4324708180aea31289c0", - "estimator_revision": "issue-7-structural-v1", + "campaign_commitment": "b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864", + "calibration_input_digest": "b3:1d6bfc9443debec09dab40fdd8b72bd78e6e5e5bf01075def337b31f92c4a104", + "estimator_revision": "needle.paired-log-ratio-power/1", "alpha_basis_points": 500, "target_power_basis_points": 9000, "routes": [ @@ -12,21 +12,22 @@ "route": "locate_implementation", "baseline_arm": "frontier_direct", "treatment_arm": "needle_miss", - "pair_key": "task_id:repetition", - "observed_log_ratio_mean": -0.4, - "observed_log_ratio_stddev": 0.2, - "required_pairs": 1 + "pair_key": "corpus_digest:campaign_commitment:task_id:route:split:repetition:pair_seed", + "observed_log_ratio_mean": -0.4337502838523616, + "observed_log_ratio_stddev": 0.10900099103037066, + "required_pairs": 3 }, { "route": "trace_state_flow", "baseline_arm": "frontier_direct", "treatment_arm": "needle_miss", - "pair_key": "task_id:repetition", - "observed_log_ratio_mean": -0.4, - "observed_log_ratio_stddev": 0.2, - "required_pairs": 1 + "pair_key": "corpus_digest:campaign_commitment:task_id:route:split:repetition:pair_seed", + "observed_log_ratio_mean": -0.4337502838523616, + "observed_log_ratio_stddev": 0.10900099103037066, + "required_pairs": 3 } ], "validated": true, - "synthetic": true + "synthetic": true, + "artifact_digest": "b3:88cadb0c43e0ac1a39566f5c76fa1a64949b2ab88bdb592aaaafc76b897b22fa" } diff --git a/benchmarks/corpus/router-cache/schedule.json b/benchmarks/corpus/router-cache/schedule.json index a187567..472f774 100644 --- a/benchmarks/corpus/router-cache/schedule.json +++ b/benchmarks/corpus/router-cache/schedule.json @@ -1,28 +1,424 @@ { "schema": "needle.corpus-schedule/1", "manifest_digest": "b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a", - "power_plan_digest": "b3:8605f040792744907aa94d3ce017a211b774c16521dbe744acb7a1b587647630", + "power_plan_digest": "b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6", "automatic_retries": false, "entries": [ - {"task_id":"ripgrep-glob-case-insensitive-locate-calibration","route":"locate_implementation","split":"calibration","arm":"frontier_direct","repetition":0}, - {"task_id":"ripgrep-glob-case-insensitive-locate-calibration","route":"locate_implementation","split":"calibration","arm":"needle_miss","repetition":0}, - {"task_id":"ripgrep-crlf-trace-calibration","route":"trace_state_flow","split":"calibration","arm":"frontier_direct","repetition":0}, - {"task_id":"ripgrep-crlf-trace-calibration","route":"trace_state_flow","split":"calibration","arm":"needle_miss","repetition":0}, - {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"frontier_direct","repetition":0}, - {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"native_subagent","repetition":0}, - {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"needle_miss","repetition":0}, - {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"exact_hit","repetition":0}, - {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"partial_hit","repetition":0}, - {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"escalation","repetition":0}, - {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"irrelevant_mutation","repetition":0}, - {"task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"relevant_mutation","repetition":0}, - {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"frontier_direct","repetition":0}, - {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"native_subagent","repetition":0}, - {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"needle_miss","repetition":0}, - {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"exact_hit","repetition":0}, - {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"partial_hit","repetition":0}, - {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"escalation","repetition":0}, - {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"irrelevant_mutation","repetition":0}, - {"task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"relevant_mutation","repetition":0} + { + "task_id": "ripgrep-glob-case-insensitive-locate-calibration", + "route": "locate_implementation", + "split": "calibration", + "arm": "frontier_direct", + "repetition": 0, + "pair_seed": 10 + }, + { + "task_id": "ripgrep-glob-case-insensitive-locate-calibration", + "route": "locate_implementation", + "split": "calibration", + "arm": "needle_miss", + "repetition": 0, + "pair_seed": 10 + }, + { + "task_id": "ripgrep-crlf-trace-calibration", + "route": "trace_state_flow", + "split": "calibration", + "arm": "frontier_direct", + "repetition": 0, + "pair_seed": 10 + }, + { + "task_id": "ripgrep-crlf-trace-calibration", + "route": "trace_state_flow", + "split": "calibration", + "arm": "needle_miss", + "repetition": 0, + "pair_seed": 10 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "frontier_direct", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "native_subagent", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "needle_miss", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "exact_hit", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "partial_hit", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "escalation", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "irrelevant_mutation", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "relevant_mutation", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "frontier_direct", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "native_subagent", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "needle_miss", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "exact_hit", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "partial_hit", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "escalation", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "irrelevant_mutation", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "relevant_mutation", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "frontier_direct", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "native_subagent", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "needle_miss", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "exact_hit", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "partial_hit", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "escalation", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "irrelevant_mutation", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-no-ignore-vcs-locate-holdout", + "route": "locate_implementation", + "split": "holdout", + "arm": "relevant_mutation", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "frontier_direct", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "native_subagent", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "needle_miss", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "exact_hit", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "partial_hit", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "escalation", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "irrelevant_mutation", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "relevant_mutation", + "repetition": 0, + "pair_seed": 100 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "frontier_direct", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "native_subagent", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "needle_miss", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "exact_hit", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "partial_hit", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "escalation", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "irrelevant_mutation", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "relevant_mutation", + "repetition": 1, + "pair_seed": 101 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "frontier_direct", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "native_subagent", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "needle_miss", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "exact_hit", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "partial_hit", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "escalation", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "irrelevant_mutation", + "repetition": 2, + "pair_seed": 102 + }, + { + "task_id": "ripgrep-null-data-trace-holdout", + "route": "trace_state_flow", + "split": "holdout", + "arm": "relevant_mutation", + "repetition": 2, + "pair_seed": 102 + } ] } diff --git a/benchmarks/corpus/router-cache/synthetic-calibration-observations.jsonl b/benchmarks/corpus/router-cache/synthetic-calibration-observations.jsonl new file mode 100644 index 0000000..7756da9 --- /dev/null +++ b/benchmarks/corpus/router-cache/synthetic-calibration-observations.jsonl @@ -0,0 +1,8 @@ +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","campaign_commitment":"b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864","task_id":"ripgrep-glob-case-insensitive-locate-calibration","route":"locate_implementation","split":"calibration","arm":"frontier_direct","repetition":0,"pair_seed":10,"quality_passed":true,"infrastructure_failure":null,"total_cost_microcredits":1000} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","campaign_commitment":"b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864","task_id":"ripgrep-glob-case-insensitive-locate-calibration","route":"locate_implementation","split":"calibration","arm":"needle_miss","repetition":0,"pair_seed":10,"quality_passed":true,"infrastructure_failure":null,"total_cost_microcredits":600} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","campaign_commitment":"b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864","task_id":"ripgrep-glob-case-insensitive-locate-calibration","route":"locate_implementation","split":"calibration","arm":"frontier_direct","repetition":1,"pair_seed":11,"quality_passed":true,"infrastructure_failure":null,"total_cost_microcredits":1000} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","campaign_commitment":"b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864","task_id":"ripgrep-glob-case-insensitive-locate-calibration","route":"locate_implementation","split":"calibration","arm":"needle_miss","repetition":1,"pair_seed":11,"quality_passed":true,"infrastructure_failure":null,"total_cost_microcredits":700} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","campaign_commitment":"b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864","task_id":"ripgrep-crlf-trace-calibration","route":"trace_state_flow","split":"calibration","arm":"frontier_direct","repetition":0,"pair_seed":10,"quality_passed":true,"infrastructure_failure":null,"total_cost_microcredits":1000} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","campaign_commitment":"b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864","task_id":"ripgrep-crlf-trace-calibration","route":"trace_state_flow","split":"calibration","arm":"needle_miss","repetition":0,"pair_seed":10,"quality_passed":true,"infrastructure_failure":null,"total_cost_microcredits":600} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","campaign_commitment":"b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864","task_id":"ripgrep-crlf-trace-calibration","route":"trace_state_flow","split":"calibration","arm":"frontier_direct","repetition":1,"pair_seed":11,"quality_passed":true,"infrastructure_failure":null,"total_cost_microcredits":1000} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","campaign_commitment":"b3:647300dac4ba3c304b4052a114f1e80b0eb214e278fd4415bbd576c369652864","task_id":"ripgrep-crlf-trace-calibration","route":"trace_state_flow","split":"calibration","arm":"needle_miss","repetition":1,"pair_seed":11,"quality_passed":true,"infrastructure_failure":null,"total_cost_microcredits":700} diff --git a/benchmarks/corpus/router-cache/synthetic-final-observations.jsonl b/benchmarks/corpus/router-cache/synthetic-final-observations.jsonl new file mode 100644 index 0000000..8fbc2d3 --- /dev/null +++ b/benchmarks/corpus/router-cache/synthetic-final-observations.jsonl @@ -0,0 +1,48 @@ +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"frontier_direct","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":1000,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"native_subagent","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"needle_miss","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":550,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"exact_hit","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":0,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"partial_hit","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":["behavior"],"expected_invalidated_nodes":["behavior"]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"escalation","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"irrelevant_mutation","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"relevant_mutation","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"frontier_direct","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":1010,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"native_subagent","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"needle_miss","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":560,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"exact_hit","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":0,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"partial_hit","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":["behavior"],"expected_invalidated_nodes":["behavior"]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"escalation","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"irrelevant_mutation","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"relevant_mutation","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"frontier_direct","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":1020,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"native_subagent","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"needle_miss","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":570,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"exact_hit","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":0,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"partial_hit","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":["behavior"],"expected_invalidated_nodes":["behavior"]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"escalation","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"irrelevant_mutation","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-no-ignore-vcs-locate-holdout","route":"locate_implementation","split":"holdout","arm":"relevant_mutation","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"frontier_direct","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":1000,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"native_subagent","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"needle_miss","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":550,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"exact_hit","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":0,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"partial_hit","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":["behavior"],"expected_invalidated_nodes":["behavior"]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"escalation","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"irrelevant_mutation","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"relevant_mutation","repetition":0,"pair_seed":100,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"frontier_direct","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":1010,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"native_subagent","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"needle_miss","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":560,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"exact_hit","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":0,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"partial_hit","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":["behavior"],"expected_invalidated_nodes":["behavior"]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"escalation","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"irrelevant_mutation","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"relevant_mutation","repetition":1,"pair_seed":101,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"frontier_direct","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":1020,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"native_subagent","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"needle_miss","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":570,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"exact_hit","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":0,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"partial_hit","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":["behavior"],"expected_invalidated_nodes":["behavior"]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"escalation","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"irrelevant_mutation","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} +{"corpus_digest":"b3:8be00a443b7e1a57813d8dbe25e6ea22a988f7536bc0f159fee8a58f0fa6cd7a","schedule_digest":"b3:3ae9b089ba4c7bee7b58ad562a5797872377ef2d986fba30d75651cc567210dd","power_plan_digest":"b3:0cfe45b5cca814731619776de15572959720cd576db2c0a8d8cf0bf8d0b561e6","task_id":"ripgrep-null-data-trace-holdout","route":"trace_state_flow","split":"holdout","arm":"relevant_mutation","repetition":2,"pair_seed":102,"quality_passed":true,"stale_hit":false,"infrastructure_failure":null,"total_cost_microcredits":100,"worker_spawns":1,"main_discovery_on_covered_scope":0,"recomputed_nodes":[],"expected_invalidated_nodes":[]} diff --git a/crates/needle-app/src/main.rs b/crates/needle-app/src/main.rs index cb4c217..45bc463 100644 --- a/crates/needle-app/src/main.rs +++ b/crates/needle-app/src/main.rs @@ -1,10 +1,14 @@ use needle_bench::{ - ArtifactStore, CachePilotArmObservation, CachePilotResolveOutcome, ExperimentArm, - ExperimentObservation, ExperimentReport, ExperimentSchedule, FinalObservation, - FrozenCorpusManifest, PilotGateResult, PricingSnapshot, ProcessExecutionStatus, ProductArm, + ArtifactStore, BootstrapConfig, CachePilotArmObservation, CachePilotResolveOutcome, + CalibrationObservation, CorpusSchedule, ExperimentArm, ExperimentObservation, ExperimentReport, + ExperimentSchedule, FinalGateContract, FinalObservation, FrozenCorpusManifest, + MAX_BOOTSTRAP_RESAMPLES, MAX_CALIBRATION_INPUT_BYTES, MAX_CORPUS_MANIFEST_BYTES, + MAX_FINAL_OBSERVATION_BYTES, MAX_SCHEDULE_BYTES, MIN_BOOTSTRAP_RESAMPLES, MultiTaskCampaign, + PilotGateResult, PowerPlan, PricingSnapshot, ProcessExecutionStatus, ProductArm, ProductObservation, ProductRunManifest, ProductVerdict, QualityOracleResult, QualityOracleSpec, TaskFixture, TokenCost, evaluate_cache_pilot, evaluate_final_gate, evaluate_mutation_pilot, - evaluate_pilot_pair, parse_codex_jsonl, parse_jsonl, parse_task_fixture, redact_jsonl, + evaluate_pilot_pair, parse_codex_jsonl, parse_jsonl, parse_task_fixture, plan_power, + raw_digest, read_bounded_file, redact_jsonl, validate_frozen_manifest, }; use needle_core::{ CodexHost, CodexRole, CommandPolicy, Digest, EvidenceFailurePolicy, FORMAT_REVISION, @@ -757,10 +761,11 @@ fn run_experiment(arguments: Vec) -> Result<(), AppError> { "cache-pilot" => product_cache_pilot_run(&arguments[1..]), "cache-pilot-report" => cache_pilot_report(&arguments[1..]), "product-report" => product_report(&arguments[1..]), + "power-plan" => power_plan_report(&arguments[1..]), "final-report" => final_gate_report(&arguments[1..]), "report" => experiment_report(&arguments[1..]), _ => Err(AppError::Usage( - "experiment run|transport-preflight|mcp-live|partial-tests-live|mcp-contract-microbench|minimal-pilot-live|quality-oracle-replay|artifact-cache-main-replay|worker-diagnostic-live|pilot|cache-pilot|cache-pilot-report|report|product-report|final-report" + "experiment run|transport-preflight|mcp-live|partial-tests-live|mcp-contract-microbench|minimal-pilot-live|quality-oracle-replay|artifact-cache-main-replay|worker-diagnostic-live|pilot|cache-pilot|cache-pilot-report|report|product-report|power-plan|final-report" .to_owned(), )), } @@ -3576,34 +3581,82 @@ fn final_gate_report(arguments: &[String]) -> Result<(), AppError> { .to_owned(), )); }; - let corpus_path = required_value(arguments, "--corpus")?; - let manifest = serde_json::from_slice::(&fs::read(corpus_path)?) - .map_err(|error| { + let corpus_path = PathBuf::from(required_value(arguments, "--corpus")?); + let manifest_bytes = read_bounded_file(&corpus_path, MAX_CORPUS_MANIFEST_BYTES)?; + let manifest = + serde_json::from_slice::(&manifest_bytes).map_err(|error| { AppError::Experiment(format!("invalid frozen corpus manifest: {error}")) })?; + let manifest_errors = validate_frozen_manifest(&manifest); + if !manifest_errors.is_empty() { + return Err(AppError::Experiment(format!( + "frozen corpus manifest is invalid: {}", + manifest_errors.join("; ") + ))); + } + let corpus_root = corpus_path.parent().unwrap_or_else(|| Path::new(".")); + let campaign_path = manifest + .campaign_path + .as_deref() + .ok_or_else(|| AppError::Experiment("frozen corpus has no campaign path".to_owned()))?; + let schedule_path = manifest + .schedule_path + .as_deref() + .ok_or_else(|| AppError::Experiment("frozen corpus has no schedule path".to_owned()))?; + let power_plan_path = manifest + .power_plan_path + .as_deref() + .ok_or_else(|| AppError::Experiment("frozen corpus has no power-plan path".to_owned()))?; + let campaign_bytes = + read_bounded_file(&corpus_root.join(campaign_path), MAX_CORPUS_MANIFEST_BYTES)?; + let schedule_bytes = read_bounded_file(&corpus_root.join(schedule_path), MAX_SCHEDULE_BYTES)?; + let power_plan_bytes = + read_bounded_file(&corpus_root.join(power_plan_path), MAX_SCHEDULE_BYTES)?; + let campaign = serde_json::from_slice::(&campaign_bytes) + .map_err(|error| AppError::Experiment(format!("invalid campaign: {error}")))?; + let schedule = serde_json::from_slice::(&schedule_bytes) + .map_err(|error| AppError::Experiment(format!("invalid corpus schedule: {error}")))?; + let power_plan = serde_json::from_slice::(&power_plan_bytes) + .map_err(|error| AppError::Experiment(format!("invalid power plan: {error}")))?; + let campaign_digest = raw_digest(&campaign_bytes); + let schedule_digest = raw_digest(&schedule_bytes); + let power_plan_digest = raw_digest(&power_plan_bytes); let resamples = option_value(arguments, "--bootstrap-resamples") .map(|value| value.parse::()) .transpose() .map_err(|error| AppError::Usage(format!("invalid bootstrap resample count: {error}")))? .unwrap_or(10_000); - if resamples < 1_000 { - return Err(AppError::Usage("--bootstrap-resamples must be at least 1000".to_owned())); + if !(MIN_BOOTSTRAP_RESAMPLES..=MAX_BOOTSTRAP_RESAMPLES).contains(&resamples) { + return Err(AppError::Usage( + "--bootstrap-resamples must be between 1000 and 1000000".to_owned(), + )); } let seed = option_value(arguments, "--seed") .map(|value| value.parse::()) .transpose() .map_err(|error| AppError::Usage(format!("invalid bootstrap seed: {error}")))? .unwrap_or(42); - let observations = fs::read_to_string(input)? + let observation_bytes = read_bounded_file(Path::new(input), MAX_FINAL_OBSERVATION_BYTES)?; + let observation_text = String::from_utf8(observation_bytes).map_err(|error| { + AppError::Experiment(format!("final observations are not UTF-8: {error}")) + })?; + let observations = observation_text .lines() .filter(|line| !line.trim().is_empty()) .map(serde_json::from_str::) .collect::, _>>()?; let rendered = serde_json::to_string_pretty(&evaluate_final_gate( - &manifest, + FinalGateContract { + manifest: &manifest, + campaign: &campaign, + schedule: &schedule, + power_plan: &power_plan, + campaign_digest: &campaign_digest, + schedule_digest: &schedule_digest, + power_plan_digest: &power_plan_digest, + }, &observations, - resamples, - seed, + BootstrapConfig { resamples, seed }, ))?; if let Some(path) = option_value(arguments, "--output") { fs::write(path, rendered)?; @@ -3613,6 +3666,47 @@ fn final_gate_report(arguments: &[String]) -> Result<(), AppError> { Ok(()) } +fn power_plan_report(arguments: &[String]) -> Result<(), AppError> { + let Some(input) = arguments.first() else { + return Err(AppError::Usage( + "experiment power-plan --corpus --campaign [--output path]" + .to_owned(), + )); + }; + let corpus_path = PathBuf::from(required_value(arguments, "--corpus")?); + let campaign_path = PathBuf::from(required_value(arguments, "--campaign")?); + let manifest_bytes = read_bounded_file(&corpus_path, MAX_CORPUS_MANIFEST_BYTES)?; + let manifest = + serde_json::from_slice::(&manifest_bytes).map_err(|error| { + AppError::Experiment(format!("invalid frozen corpus manifest: {error}")) + })?; + let campaign_bytes = read_bounded_file(&campaign_path, MAX_CORPUS_MANIFEST_BYTES)?; + if manifest.campaign_digest.as_deref() != Some(raw_digest(&campaign_bytes).as_str()) { + return Err(AppError::Experiment( + "campaign bytes differ from the frozen manifest".to_owned(), + )); + } + let campaign = serde_json::from_slice::(&campaign_bytes) + .map_err(|error| AppError::Experiment(format!("invalid campaign: {error}")))?; + let observation_bytes = read_bounded_file(Path::new(input), MAX_CALIBRATION_INPUT_BYTES)?; + let observation_text = String::from_utf8(observation_bytes).map_err(|error| { + AppError::Experiment(format!("calibration observations are not UTF-8: {error}")) + })?; + let observations = observation_text + .lines() + .filter(|line| !line.trim().is_empty()) + .map(serde_json::from_str::) + .collect::, _>>()?; + let report = plan_power(&manifest, &campaign, &observations); + let rendered = serde_json::to_string_pretty(&report)?; + if let Some(path) = option_value(arguments, "--output") { + fs::write(path, rendered)?; + } else { + println!("{rendered}"); + } + Ok(()) +} + fn option_value(arguments: &[String], name: &str) -> Option { arguments.windows(2).find(|pair| pair[0] == name).map(|pair| pair[1].clone()) } diff --git a/crates/needle-bench/src/corpus.rs b/crates/needle-bench/src/corpus.rs index 8e11bff..0ed74a6 100644 --- a/crates/needle-bench/src/corpus.rs +++ b/crates/needle-bench/src/corpus.rs @@ -828,7 +828,7 @@ fn load_multi_task_campaign( errors.push("v2 campaign requires a validated schedule and power plan".to_owned()); return None; }; - if power_plan.campaign_digest != campaign_commitment(&campaign) { + if power_plan.campaign_commitment != campaign_commitment(&campaign) { errors.push("power plan campaign commitment differs from campaign".to_owned()); return None; } @@ -1265,7 +1265,7 @@ mod tests { assert!(report.campaign_valid); assert_eq!( report.cost_readiness.full_protocol_campaign_budget_estimate_microcredits, - Some(172_157_430) + Some(429_519_010) ); assert!(!report.provider_inputs_ready); assert!(!report.provider_run_ready); diff --git a/crates/needle-bench/src/final_gate.rs b/crates/needle-bench/src/final_gate.rs index 7dd32a5..d3da5dd 100644 --- a/crates/needle-bench/src/final_gate.rs +++ b/crates/needle-bench/src/final_gate.rs @@ -2,8 +2,16 @@ use serde::{Deserialize, Serialize}; use statrs::distribution::{ContinuousCDF, Normal}; use std::collections::{BTreeMap, BTreeSet}; +use crate::{ + CorpusSchedule, MAX_SCHEDULE_ENTRIES, MultiTaskCampaign, PowerPlan, PowerRoutePlan, + campaign_commitment, validate_power_campaign, +}; + pub const MAX_CORPUS_TASKS: usize = 512; pub const MAX_CORPUS_MANIFEST_BYTES: usize = 1024 * 1024; +pub const MAX_FINAL_OBSERVATION_BYTES: usize = 64 * 1024 * 1024; +pub const MIN_BOOTSTRAP_RESAMPLES: usize = 1_000; +pub const MAX_BOOTSTRAP_RESAMPLES: usize = 1_000_000; const MAX_CORPUS_IDENTIFIER_BYTES: usize = 256; #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] @@ -135,6 +143,9 @@ pub struct FrozenCorpusManifest { #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct FinalObservation { + pub corpus_digest: String, + pub schedule_digest: String, + pub power_plan_digest: String, pub task_id: String, pub route: BenchmarkRoute, pub split: CorpusSplit, @@ -164,7 +175,10 @@ pub struct RouteGate { pub valid_pairs: usize, pub required_pairs: Option, pub powered: bool, + pub calibration_log_ratio_mean: Option, + pub calibration_log_ratio_stddev: Option, pub infrastructure_failures: usize, + pub validation_failures: Vec, pub passed: bool, } @@ -172,82 +186,133 @@ pub struct RouteGate { pub struct FinalGateReport { pub schema: String, pub corpus_digest: String, + pub campaign_digest: String, + pub campaign_commitment: String, + pub schedule_digest: String, + pub power_plan_digest: String, + pub power_plan_artifact_digest: String, + pub estimator_revision: String, + pub alpha_basis_points: u16, + pub target_power_basis_points: u16, + pub bootstrap_seed: u64, + pub bootstrap_resamples: usize, pub economic_baseline: FinalArm, pub economic_treatment: FinalArm, pub manifest_valid: bool, pub manifest_errors: Vec, + pub contract_valid: bool, + pub validation_failures: Vec, pub routes: Vec, pub passed: bool, } -#[derive(Clone, Debug, PartialEq)] -pub struct PowerEstimate { - pub observed_log_ratio_mean: f64, - pub observed_log_ratio_stddev: f64, - pub required_pairs_per_route: usize, - pub power: f64, - pub one_sided_alpha: f64, +#[derive(Clone, Copy, Debug)] +pub struct FinalGateContract<'a> { + pub manifest: &'a FrozenCorpusManifest, + pub campaign: &'a MultiTaskCampaign, + pub schedule: &'a CorpusSchedule, + pub power_plan: &'a PowerPlan, + pub campaign_digest: &'a str, + pub schedule_digest: &'a str, + pub power_plan_digest: &'a str, } -pub fn estimate_required_pairs(calibration_ratios: &[f64]) -> Option { - if calibration_ratios.len() < 2 - || calibration_ratios.iter().any(|ratio| !ratio.is_finite() || *ratio <= 0.0) - { - return None; - } - let logs = calibration_ratios.iter().map(|ratio| ratio.ln()).collect::>(); - let mean = average(&logs); - if mean >= 0.0 { - return None; - } - let variance = - logs.iter().map(|value| (value - mean).powi(2)).sum::() / (logs.len() - 1) as f64; - let stddev = variance.sqrt(); - if !stddev.is_finite() || stddev == 0.0 { - return None; - } - let required = (((1.644_853_626_951_472_2 + 1.281_551_565_544_600_4) * stddev / mean.abs()) - .powi(2)) - .ceil() - .max(2.0) as usize; - Some(PowerEstimate { - observed_log_ratio_mean: mean, - observed_log_ratio_stddev: stddev, - required_pairs_per_route: required, - power: 0.90, - one_sided_alpha: 0.05, - }) +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct BootstrapConfig { + pub resamples: usize, + pub seed: u64, } pub fn evaluate_final_gate( - manifest: &FrozenCorpusManifest, + contract: FinalGateContract<'_>, observations: &[FinalObservation], - bootstrap_resamples: usize, - bootstrap_seed: u64, + bootstrap: BootstrapConfig, ) -> FinalGateReport { - let mut manifest_errors = validate_frozen_manifest(manifest); - if manifest.schema == "needle.frozen-corpus/4" { - manifest_errors.push( - "v4 final gate is unavailable until schedule-bound exact observations are integrated" - .to_owned(), - ); - } - manifest_errors.extend(validate_observations(manifest, observations)); + let FinalGateContract { + manifest, + campaign, + schedule, + power_plan, + campaign_digest, + schedule_digest, + power_plan_digest, + } = contract; + let BootstrapConfig { resamples: bootstrap_resamples, seed: bootstrap_seed } = bootstrap; + let manifest_errors = validate_frozen_manifest(manifest); let manifest_valid = manifest_errors.is_empty(); + let mut validation_failures = Vec::new(); + if manifest.schema != "needle.frozen-corpus/4" { + validation_failures.push("final gate requires frozen corpus v4".to_owned()); + } + if !(MIN_BOOTSTRAP_RESAMPLES..=MAX_BOOTSTRAP_RESAMPLES).contains(&bootstrap_resamples) { + validation_failures.push("bootstrap resample count is outside bounded limits".to_owned()); + } + validation_failures.extend(validate_power_campaign(campaign)); + let expected_campaign_commitment = campaign_commitment(campaign); + if manifest.campaign_digest.as_deref() != Some(campaign_digest) + || !valid_blake3_digest(campaign_digest) + { + validation_failures.push("campaign digest differs from the frozen manifest".to_owned()); + } + if power_plan.campaign_commitment != expected_campaign_commitment { + validation_failures.push("power plan campaign commitment differs".to_owned()); + } + if power_plan.synthetic { + validation_failures + .push("synthetic power plan is ineligible for the final claim".to_owned()); + } + if manifest.schedule_digest.as_deref() != Some(schedule_digest) + || !valid_blake3_digest(schedule_digest) + { + validation_failures.push("schedule digest differs from the frozen manifest".to_owned()); + } + if manifest.power_plan_digest.as_deref() != Some(power_plan_digest) + || !valid_blake3_digest(power_plan_digest) + { + validation_failures.push("power plan digest differs from the frozen manifest".to_owned()); + } + validation_failures.extend(schedule.validate(manifest, power_plan, power_plan_digest)); + validation_failures.extend(validate_observations( + manifest, + schedule, + schedule_digest, + power_plan_digest, + observations, + )); + let contract_valid = manifest_valid && validation_failures.is_empty(); let routes = [BenchmarkRoute::LocateImplementation, BenchmarkRoute::TraceStateFlow] .into_iter() - .map(|route| evaluate_route(observations, route, bootstrap_resamples, bootstrap_seed)) + .map(|route| { + evaluate_route( + observations, + power_plan.routes.iter().find(|plan| plan.route == route), + route, + contract_valid, + bootstrap_resamples, + bootstrap_seed, + ) + }) .collect::>(); FinalGateReport { - schema: "needle.final-gate/2".to_owned(), - corpus_digest: blake3::hash(&serde_json::to_vec(manifest).unwrap_or_default()) - .to_hex() - .to_string(), + schema: "needle.final-gate/3".to_owned(), + corpus_digest: crate::corpus_digest(manifest), + campaign_digest: campaign_digest.to_owned(), + campaign_commitment: expected_campaign_commitment, + schedule_digest: schedule_digest.to_owned(), + power_plan_digest: power_plan_digest.to_owned(), + power_plan_artifact_digest: power_plan.artifact_digest.clone(), + estimator_revision: power_plan.estimator_revision.clone(), + alpha_basis_points: power_plan.alpha_basis_points, + target_power_basis_points: power_plan.target_power_basis_points, + bootstrap_seed, + bootstrap_resamples, economic_baseline: FinalArm::FrontierDirect, economic_treatment: FinalArm::NeedleMiss, manifest_valid, manifest_errors, - passed: manifest_valid && routes.iter().all(|route| route.passed), + contract_valid, + validation_failures, + passed: contract_valid && routes.iter().all(|route| route.passed), routes, } } @@ -433,37 +498,103 @@ fn validate_v4_manifest(manifest: &FrozenCorpusManifest, errors: &mut Vec Vec { let mut errors = Vec::new(); - let tasks = - manifest.tasks.iter().map(|task| (task.id.as_str(), task)).collect::>(); - let mut observation_keys = BTreeSet::new(); + if observations.len() > MAX_SCHEDULE_ENTRIES { + return vec!["final observation count exceeds bounded maximum".to_owned()]; + } + let manifest_digest = crate::corpus_digest(manifest); + let expected = schedule + .entries + .iter() + .filter(|entry| entry.split == CorpusSplit::Holdout) + .map(|entry| { + ( + entry.task_id.clone(), + entry.route, + entry.split, + entry.arm, + entry.repetition, + entry.pair_seed, + ) + }) + .collect::>(); + let mut observed = BTreeSet::new(); for observation in observations { - let Some(task) = tasks.get(observation.task_id.as_str()) else { - errors.push(format!("observation references unknown task `{}`", observation.task_id)); - continue; - }; - if task.route != observation.route || task.split != observation.split { - errors - .push(format!("observation metadata differs from task `{}`", observation.task_id)); + if observation.corpus_digest != manifest_digest + || observation.schedule_digest != schedule_digest + || observation.power_plan_digest != power_plan_digest + { + errors.push(format!( + "observation `{}` has stale corpus, schedule, or power-plan identity", + observation.task_id + )); + } + if observation.split != CorpusSplit::Holdout { + errors.push(format!( + "calibration observation `{}` cannot enter the final gate", + observation.task_id + )); } let key = ( - observation.task_id.as_str(), + observation.task_id.clone(), + observation.route, + observation.split, observation.arm, observation.repetition, observation.pair_seed, ); - if !observation_keys.insert(key) { + if !observed.insert(key.clone()) { errors .push(format!("duplicate observation identity for task `{}`", observation.task_id)); } + if !expected.contains(&key) { + errors.push(format!( + "observation identity for task `{}` is not present in the frozen schedule", + observation.task_id + )); + } + if observation.infrastructure_failure.as_ref().is_some_and(|detail| detail.len() > 512) + || observation.recomputed_nodes.len() > 512 + || observation.expected_invalidated_nodes.len() > 512 + || observation + .recomputed_nodes + .iter() + .chain(&observation.expected_invalidated_nodes) + .any(|node| node.is_empty() || node.len() > 512) + { + errors.push(format!( + "observation `{}` exceeds bounded evidence limits", + observation.task_id + )); + } } - for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Holdout) { - for arm in FinalArm::ALL { - if !observations.iter().any(|item| item.task_id == task.id && item.arm == arm) { - errors.push(format!("holdout task `{}` is missing arm {arm:?}", task.id)); - } + for missing in expected.difference(&observed) { + errors.push(format!( + "scheduled holdout observation is missing for task `{}` arm {:?} repetition {} seed {}", + missing.0, missing.3, missing.4, missing.5 + )); + } + if expected.len() + != schedule.entries.iter().filter(|entry| entry.split == CorpusSplit::Holdout).count() + { + errors.push("frozen schedule contains duplicate holdout identities".to_owned()); + } + let task_metadata = manifest + .tasks + .iter() + .map(|task| (task.id.as_str(), (task.route, task.split))) + .collect::>(); + for observation in observations { + if task_metadata.get(observation.task_id.as_str()) + != Some(&(observation.route, observation.split)) + { + errors + .push(format!("observation metadata differs from task `{}`", observation.task_id)); } } errors @@ -487,7 +618,9 @@ fn safe_relative_json_path(value: &str) -> bool { fn evaluate_route( observations: &[FinalObservation], + power_plan: Option<&PowerRoutePlan>, route: BenchmarkRoute, + contract_valid: bool, bootstrap_resamples: usize, bootstrap_seed: u64, ) -> RouteGate { @@ -495,40 +628,75 @@ fn evaluate_route( .iter() .filter(|item| item.route == route && item.split == CorpusSplit::Holdout) .collect::>(); + let mut validation_failures = Vec::new(); + if !contract_valid { + validation_failures + .push("global manifest, plan, schedule, or observation contract failed".to_owned()); + } let infrastructure_failures = relevant.iter().filter(|item| item.infrastructure_failure.is_some()).count(); - let valid = relevant - .iter() - .filter(|item| item.infrastructure_failure.is_none()) - .copied() - .collect::>(); - let quality_passed = !valid.is_empty() && valid.iter().all(|item| item.quality_passed); - let stale_safe = !valid.is_empty() && valid.iter().all(|item| !item.stale_hit); - let exact = valid.iter().filter(|item| item.arm == FinalArm::ExactHit).collect::>(); + for item in &relevant { + if let Some(detail) = item.infrastructure_failure.as_deref() { + let detail = if detail.len() <= 512 { detail } else { "" }; + validation_failures.push(format!( + "task `{}` arm {:?} has an infrastructure failure: {detail}", + item.task_id, item.arm, + )); + } + if !item.quality_passed { + validation_failures + .push(format!("task `{}` arm {:?} failed quality", item.task_id, item.arm)); + } + if item.stale_hit { + validation_failures + .push(format!("task `{}` arm {:?} produced a stale hit", item.task_id, item.arm)); + } + } + let quality_passed = !relevant.is_empty() && relevant.iter().all(|item| item.quality_passed); + let stale_safe = !relevant.is_empty() && relevant.iter().all(|item| !item.stale_hit); + let exact = relevant.iter().filter(|item| item.arm == FinalArm::ExactHit).collect::>(); let exact_zero_worker = !exact.is_empty() && exact.iter().all(|item| item.worker_spawns == 0); - let partial = valid.iter().filter(|item| item.arm == FinalArm::PartialHit).collect::>(); + if !exact_zero_worker { + validation_failures.push("exact-hit observations must spawn zero workers".to_owned()); + } + let partial = + relevant.iter().filter(|item| item.arm == FinalArm::PartialHit).collect::>(); let partial_exact_recomputation = !partial.is_empty() && partial.iter().all(|item| { let actual = item.recomputed_nodes.iter().collect::>(); let expected = item.expected_invalidated_nodes.iter().collect::>(); - actual == expected + actual.len() == item.recomputed_nodes.len() + && expected.len() == item.expected_invalidated_nodes.len() + && actual == expected }); - let zero_main_discovery = - !valid.is_empty() && valid.iter().all(|item| item.main_discovery_on_covered_scope == 0); - let ratios = paired_ratios(&valid); - let calibration = observations - .iter() - .filter(|item| { - item.route == route - && item.split == CorpusSplit::Calibration - && item.infrastructure_failure.is_none() - }) - .collect::>(); - let power = estimate_required_pairs(&paired_ratios(&calibration)); - let required_pairs = power.as_ref().map(|estimate| estimate.required_pairs_per_route); - let powered = required_pairs.is_some_and(|required| ratios.len() >= required); - let paired_cost_ratio = (!ratios.is_empty()).then(|| average(&ratios)); - let bca_95_ci = bca_interval(&ratios, bootstrap_resamples, bootstrap_seed); + if !partial_exact_recomputation { + validation_failures + .push("partial-hit invalidated nodes differ from recomputed nodes".to_owned()); + } + let zero_main_discovery = !relevant.is_empty() + && relevant.iter().all(|item| item.main_discovery_on_covered_scope == 0); + if !zero_main_discovery { + validation_failures.push("covered-scope observation performed main discovery".to_owned()); + } + let (ratios, pair_failures) = paired_ratios(&relevant); + validation_failures.extend(pair_failures); + let required_pairs = power_plan.map(|plan| plan.required_pairs as usize); + if power_plan.is_none() { + validation_failures.push(format!("power plan is missing route {route:?}")); + } + let powered = required_pairs == Some(ratios.len()); + if !powered { + validation_failures.push(format!( + "route {route:?} observed {} valid pairs but requires {:?}", + ratios.len(), + required_pairs + )); + } + let statistical_inputs_valid = validation_failures.is_empty() && powered; + let paired_cost_ratio = statistical_inputs_valid.then(|| average(&ratios)); + let bca_95_ci = statistical_inputs_valid + .then(|| bca_interval(&ratios, bootstrap_resamples, bootstrap_seed)) + .flatten(); let economics = bca_95_ci.is_some_and(|interval| interval[1] < 1.0); RouteGate { route, @@ -542,36 +710,55 @@ fn evaluate_route( valid_pairs: ratios.len(), required_pairs, powered, + calibration_log_ratio_mean: power_plan.map(|plan| plan.observed_log_ratio_mean), + calibration_log_ratio_stddev: power_plan.map(|plan| plan.observed_log_ratio_stddev), infrastructure_failures, + validation_failures, passed: quality_passed && stale_safe && exact_zero_worker && partial_exact_recomputation && zero_main_discovery && powered - && economics, + && economics + && statistical_inputs_valid, } } -fn paired_ratios(observations: &[&FinalObservation]) -> Vec { +fn paired_ratios(observations: &[&FinalObservation]) -> (Vec, Vec) { let mut pairs = BTreeMap::<(String, u32, u64), (Option, Option)>::new(); + let mut errors = Vec::new(); for observation in observations { let key = (observation.task_id.clone(), observation.repetition, observation.pair_seed); let pair = pairs.entry(key).or_default(); match observation.arm { - FinalArm::FrontierDirect => pair.0 = observation.total_cost_microcredits, - FinalArm::NeedleMiss => pair.1 = observation.total_cost_microcredits, + FinalArm::FrontierDirect => { + if pair.0.is_some() { + errors.push(format!("duplicate baseline for task `{}`", observation.task_id)); + } + pair.0 = observation.total_cost_microcredits; + } + FinalArm::NeedleMiss => { + if pair.1.is_some() { + errors.push(format!("duplicate treatment for task `{}`", observation.task_id)); + } + pair.1 = observation.total_cost_microcredits; + } _ => {} } } - pairs - .into_values() - .filter_map(|(native, needle)| { - let native = native?; - let needle = needle?; - (native > 0).then_some(needle as f64 / native as f64) - }) - .collect() + let mut ratios = Vec::new(); + for ((task_id, repetition, pair_seed), (baseline, treatment)) in pairs { + match (baseline, treatment) { + (Some(baseline), Some(treatment)) if baseline > 0 && treatment > 0 => { + ratios.push(treatment as f64 / baseline as f64); + } + _ => errors.push(format!( + "economic pair `{task_id}` repetition {repetition} seed {pair_seed} is incomplete or non-positive" + )), + } + } + (ratios, errors) } fn bca_interval(values: &[f64], resamples: usize, seed: u64) -> Option<[f64; 2]> { @@ -648,237 +835,5 @@ impl Lcg { } #[cfg(test)] -mod tests { - use super::*; - - fn manifest() -> FrozenCorpusManifest { - let mut tasks = Vec::new(); - for route in [BenchmarkRoute::LocateImplementation, BenchmarkRoute::TraceStateFlow] { - for split in [CorpusSplit::Calibration, CorpusSplit::Holdout] { - tasks.push(CorpusTask { - id: if split == CorpusSplit::Calibration { - format!("{route:?}-cal") - } else { - format!("{route:?}") - }, - route, - split, - repository_url: "https://example.invalid/repository.git".to_owned(), - repository_sha: "0123456789abcdef0123456789abcdef01234567".to_owned(), - prompt: - "Locate the implementation and provide a focused test for this behavior." - .to_owned(), - material_class: CorpusMaterialClass::Legacy, - focused_test_policy: FocusedTestPolicyRef::default(), - oracle_schema: String::new(), - oracle_path: format!("oracles/{route:?}-{split:?}.json"), - oracle_digest: - "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" - .to_owned(), - test_identifier: "misc::example".to_owned(), - focused_command: [ - "cargo", - "test", - "--offline", - "--test", - "integration", - "misc::example", - "--", - "--exact", - ] - .into_iter() - .map(str::to_owned) - .collect(), - }); - } - } - FrozenCorpusManifest { - schema: "needle.frozen-corpus/2".to_owned(), - frozen_unix_ms: 1, - arms: FinalArm::ALL.to_vec(), - cost_model_path: "cost-model.json".to_owned(), - cost_model_digest: - "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), - next_pilot_path: "minimal-live-pilot.json".to_owned(), - next_pilot_digest: - "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), - campaign_path: None, - campaign_digest: None, - schedule_path: None, - schedule_digest: None, - power_plan_path: None, - power_plan_digest: None, - sealed_bundle_schema: None, - sealed_bundle_digest: None, - tasks, - } - } - - fn observation( - route: BenchmarkRoute, - arm: FinalArm, - repetition: u32, - cost: Option, - ) -> FinalObservation { - FinalObservation { - task_id: format!("{route:?}"), - route, - split: CorpusSplit::Holdout, - arm, - repetition, - pair_seed: repetition as u64, - quality_passed: true, - stale_hit: false, - infrastructure_failure: None, - total_cost_microcredits: cost, - worker_spawns: u32::from(arm != FinalArm::ExactHit), - main_discovery_on_covered_scope: 0, - recomputed_nodes: if arm == FinalArm::PartialHit { - vec!["behavior".to_owned()] - } else { - Vec::new() - }, - expected_invalidated_nodes: if arm == FinalArm::PartialHit { - vec!["behavior".to_owned()] - } else { - Vec::new() - }, - } - } - - #[test] - fn bca_gate_passes_only_when_both_routes_have_upper_bound_below_one() { - let mut observations = Vec::new(); - for route in [BenchmarkRoute::LocateImplementation, BenchmarkRoute::TraceStateFlow] { - for arm in [ - FinalArm::NativeSubagent, - FinalArm::ExactHit, - FinalArm::PartialHit, - FinalArm::Escalation, - FinalArm::IrrelevantMutation, - FinalArm::RelevantMutation, - ] { - observations.push(observation(route, arm, 0, None)); - } - for repetition in 1..=6 { - let mut frontier = - observation(route, FinalArm::FrontierDirect, repetition, Some(1_000)); - frontier.task_id = format!("{route:?}-cal"); - frontier.split = CorpusSplit::Calibration; - observations.push(frontier); - let mut needle = observation( - route, - FinalArm::NeedleMiss, - repetition, - Some(600 + repetition as u64 * 10), - ); - needle.task_id = format!("{route:?}-cal"); - needle.split = CorpusSplit::Calibration; - observations.push(needle); - } - for repetition in 1..=12 { - observations.push(observation( - route, - FinalArm::FrontierDirect, - repetition, - Some(1_000 + repetition as u64), - )); - observations.push(observation( - route, - FinalArm::NeedleMiss, - repetition, - Some(600 + repetition as u64), - )); - } - } - let report = evaluate_final_gate(&manifest(), &observations, 2_000, 7); - assert!(report.passed); - assert!(report.manifest_valid); - assert_eq!(report.economic_baseline, FinalArm::FrontierDirect); - assert_eq!(report.economic_treatment, FinalArm::NeedleMiss); - assert!(report.routes.iter().all(|route| route.powered)); - assert!(report.routes.iter().all(|route| route.bca_95_ci.unwrap()[1] < 1.0)); - } - - #[test] - fn any_stale_or_quality_failure_blocks_the_route() { - let mut observations = vec![ - observation(BenchmarkRoute::LocateImplementation, FinalArm::ExactHit, 0, None), - observation(BenchmarkRoute::LocateImplementation, FinalArm::PartialHit, 0, None), - ]; - observations[0].stale_hit = true; - let report = evaluate_final_gate(&manifest(), &observations, 2_000, 1); - assert!(!report.routes[0].passed); - assert!(!report.routes[0].stale_safe); - } - - #[test] - fn incomplete_manifest_or_missing_arms_cannot_pass() { - let observations = vec![ - observation(BenchmarkRoute::LocateImplementation, FinalArm::ExactHit, 0, None), - observation(BenchmarkRoute::LocateImplementation, FinalArm::PartialHit, 0, None), - ]; - let report = evaluate_final_gate(&manifest(), &observations, 2_000, 9); - assert!(!report.manifest_valid); - assert!(!report.passed); - assert!(report.manifest_errors.iter().any(|error| error.contains("missing arm"))); - } - - #[test] - fn frozen_corpus_v3_requires_a_content_addressed_campaign() { - let mut manifest = manifest(); - manifest.schema = "needle.frozen-corpus/3".to_owned(); - assert!(validate_frozen_manifest(&manifest).iter().any(|error| { - error == "frozen corpus v3 requires a valid multi-task campaign reference" - })); - - manifest.campaign_path = Some("campaign.json".to_owned()); - manifest.campaign_digest = - Some("b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned()); - assert!(validate_frozen_manifest(&manifest).is_empty()); - } - - #[test] - fn legacy_answer_fields_round_trip_but_v4_rejects_them() { - let legacy = manifest(); - let encoded = serde_json::to_vec(&legacy).expect("legacy manifest serializes"); - let decoded: FrozenCorpusManifest = - serde_json::from_slice(&encoded).expect("legacy manifest round-trips"); - assert_eq!(decoded, legacy); - - let mut v4 = legacy; - v4.schema = "needle.frozen-corpus/4".to_owned(); - v4.tasks.iter_mut().for_each(|task| { - task.material_class = CorpusMaterialClass::Synthetic; - task.oracle_path.clear(); - task.test_identifier.clear(); - task.focused_command.clear(); - }); - let mut contaminated = v4.clone(); - contaminated.tasks[0].oracle_path = "legacy.json".to_owned(); - let errors = validate_frozen_manifest(&contaminated); - assert!(errors.iter().any(|error| error.contains("legacy answer-bearing"))); - } - - #[test] - fn v4_final_gate_is_explicitly_fail_closed() { - let mut manifest = manifest(); - manifest.schema = "needle.frozen-corpus/4".to_owned(); - let report = evaluate_final_gate(&manifest, &[], 1_000, 1); - assert!(!report.passed); - assert!( - report - .manifest_errors - .iter() - .any(|error| error.contains("schedule-bound exact observations")) - ); - } - - #[test] - fn power_estimate_uses_only_observed_calibration_variance() { - let estimate = estimate_required_pairs(&[0.70, 0.76, 0.82, 0.73]).unwrap(); - assert_eq!(estimate.power, 0.90); - assert_eq!(estimate.one_sided_alpha, 0.05); - assert!(estimate.required_pairs_per_route >= 2); - } -} +#[path = "final_gate/tests.rs"] +mod current_tests; diff --git a/crates/needle-bench/src/final_gate/tests.rs b/crates/needle-bench/src/final_gate/tests.rs new file mode 100644 index 0000000..15672d4 --- /dev/null +++ b/crates/needle-bench/src/final_gate/tests.rs @@ -0,0 +1,510 @@ +use super::*; +use crate::{ + CORPUS_SCHEDULE_SCHEMA, CalibrationObservation, CampaignBudgetReserve, CorpusMaterialClass, + CorpusSchedule, CorpusScheduleEntry, FocusedTestPolicyRef, MultiTaskCampaign, + POWER_PLAN_ESTIMATOR_REVISION, POWER_PLAN_PAIR_KEY, POWER_PLAN_SCHEMA, PowerPlan, + campaign_commitment, corpus_digest, plan_power, raw_digest, +}; + +fn digest(byte: char) -> String { + format!("b3:{}", byte.to_string().repeat(64)) +} + +fn task( + id: &str, + route: BenchmarkRoute, + split: CorpusSplit, + material_class: CorpusMaterialClass, +) -> CorpusTask { + CorpusTask { + id: id.to_owned(), + route, + split, + repository_url: "https://example.invalid/repository.git".to_owned(), + repository_sha: "0123456789abcdef0123456789abcdef01234567".to_owned(), + prompt: "A bounded final-gate benchmark prompt with no embedded answer material." + .to_owned(), + material_class, + focused_test_policy: FocusedTestPolicyRef { + identity: format!("policy-{id}"), + commitment: digest('1'), + }, + oracle_schema: "needle.sealed-oracle/1".to_owned(), + oracle_digest: digest('2'), + oracle_path: String::new(), + test_identifier: String::new(), + focused_command: Vec::new(), + } +} + +fn manifest(material_class: CorpusMaterialClass) -> FrozenCorpusManifest { + FrozenCorpusManifest { + schema: "needle.frozen-corpus/4".to_owned(), + frozen_unix_ms: 1, + arms: FinalArm::ALL.to_vec(), + cost_model_path: "cost-model.json".to_owned(), + cost_model_digest: digest('3'), + next_pilot_path: "pilot.json".to_owned(), + next_pilot_digest: digest('4'), + campaign_path: Some("campaign.json".to_owned()), + campaign_digest: Some(digest('5')), + schedule_path: Some("schedule.json".to_owned()), + schedule_digest: None, + power_plan_path: Some("power-plan.json".to_owned()), + power_plan_digest: None, + sealed_bundle_schema: Some("needle.sealed-oracle-index/1".to_owned()), + sealed_bundle_digest: Some(digest('8')), + tasks: vec![ + task( + "cal-locate", + BenchmarkRoute::LocateImplementation, + CorpusSplit::Calibration, + material_class, + ), + task( + "cal-trace", + BenchmarkRoute::TraceStateFlow, + CorpusSplit::Calibration, + material_class, + ), + task( + "hold-locate", + BenchmarkRoute::LocateImplementation, + CorpusSplit::Holdout, + material_class, + ), + task( + "hold-trace", + BenchmarkRoute::TraceStateFlow, + CorpusSplit::Holdout, + material_class, + ), + ], + } +} + +fn campaign() -> MultiTaskCampaign { + MultiTaskCampaign { + schema: "needle.multi-task-campaign/2".to_owned(), + schedule_digest: digest('6'), + task_ids: Vec::new(), + paid_arms: vec![FinalArm::FrontierDirect, FinalArm::NeedleMiss], + offline_cache_arms: vec![ + FinalArm::ExactHit, + FinalArm::PartialHit, + FinalArm::IrrelevantMutation, + FinalArm::RelevantMutation, + ], + deferred_diagnostic_arms: vec![FinalArm::NativeSubagent, FinalArm::Escalation], + repetitions_per_task: 0, + automatic_retries: false, + statistical_claim: false, + bootstrap_resamples: 10_000, + one_sided_alpha_basis_points: 500, + target_power_basis_points: 9_000, + budget_reserve: CampaignBudgetReserve { + main_turn_microcredits: 1, + extra_main_turns_per_needle_observation: 1, + worker_microcredits: 1, + extra_workers_per_needle_observation: 0, + evidence: vec!["synthetic".to_owned()], + }, + } +} + +fn calibration_observations( + manifest: &FrozenCorpusManifest, + campaign: &MultiTaskCampaign, +) -> Vec { + let corpus = corpus_digest(manifest); + let campaign = campaign_commitment(campaign); + let mut observations = Vec::new(); + for (task_id, route) in [ + ("cal-locate", BenchmarkRoute::LocateImplementation), + ("cal-trace", BenchmarkRoute::TraceStateFlow), + ] { + for (repetition, treatment) in [(0, 600), (1, 700)] { + for (arm, cost) in + [(FinalArm::FrontierDirect, 1_000), (FinalArm::NeedleMiss, treatment)] + { + observations.push(CalibrationObservation { + corpus_digest: corpus.clone(), + campaign_commitment: campaign.clone(), + task_id: task_id.to_owned(), + route, + split: CorpusSplit::Calibration, + arm, + repetition, + pair_seed: 10 + repetition as u64, + quality_passed: true, + infrastructure_failure: None, + total_cost_microcredits: Some(cost), + }); + } + } + } + observations +} + +struct Fixture { + manifest: FrozenCorpusManifest, + campaign: MultiTaskCampaign, + campaign_digest: String, + schedule: CorpusSchedule, + schedule_digest: String, + plan: PowerPlan, + plan_digest: String, + observations: Vec, +} + +fn fixture(material_class: CorpusMaterialClass) -> Fixture { + let mut manifest = manifest(material_class); + let campaign = campaign(); + let campaign_digest = raw_digest(&serde_json::to_vec(&campaign).unwrap()); + manifest.campaign_digest = Some(campaign_digest.clone()); + let planning = + plan_power(&manifest, &campaign, &calibration_observations(&manifest, &campaign)); + assert!(planning.failures.is_empty(), "{:?}", planning.failures); + let plan = planning.plan.unwrap(); + let plan_digest = raw_digest(&serde_json::to_vec(&plan).unwrap()); + let mut entries = Vec::new(); + for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Calibration) { + for arm in [FinalArm::FrontierDirect, FinalArm::NeedleMiss] { + entries.push(CorpusScheduleEntry { + task_id: task.id.clone(), + route: task.route, + split: task.split, + arm, + repetition: 0, + pair_seed: 10, + }); + } + } + for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Holdout) { + for repetition in 0..plan.required_pairs(task.route).unwrap() { + for arm in FinalArm::ALL { + entries.push(CorpusScheduleEntry { + task_id: task.id.clone(), + route: task.route, + split: task.split, + arm, + repetition, + pair_seed: 100 + repetition as u64, + }); + } + } + } + let schedule = CorpusSchedule { + schema: CORPUS_SCHEDULE_SCHEMA.to_owned(), + manifest_digest: corpus_digest(&manifest), + power_plan_digest: plan_digest.clone(), + automatic_retries: false, + entries, + }; + let schedule_digest = raw_digest(&serde_json::to_vec(&schedule).unwrap()); + manifest.schedule_digest = Some(schedule_digest.clone()); + manifest.power_plan_digest = Some(plan_digest.clone()); + assert!(validate_frozen_manifest(&manifest).is_empty()); + assert!(plan.validate(&manifest).is_empty()); + assert!(schedule.validate(&manifest, &plan, &plan_digest).is_empty()); + + let corpus = corpus_digest(&manifest); + let observations = schedule + .entries + .iter() + .filter(|entry| entry.split == CorpusSplit::Holdout) + .map(|entry| { + let invalidated_nodes = if entry.arm == FinalArm::PartialHit { + vec!["behavior".to_owned()] + } else { + Vec::new() + }; + FinalObservation { + corpus_digest: corpus.clone(), + schedule_digest: schedule_digest.clone(), + power_plan_digest: plan_digest.clone(), + task_id: entry.task_id.clone(), + route: entry.route, + split: entry.split, + arm: entry.arm, + repetition: entry.repetition, + pair_seed: entry.pair_seed, + quality_passed: true, + stale_hit: false, + infrastructure_failure: None, + total_cost_microcredits: match entry.arm { + FinalArm::FrontierDirect => Some(1_000 + entry.repetition as u64 * 10), + FinalArm::NeedleMiss => Some(550 + entry.repetition as u64 * 10), + _ => Some(100), + }, + worker_spawns: u32::from(entry.arm != FinalArm::ExactHit), + main_discovery_on_covered_scope: 0, + recomputed_nodes: invalidated_nodes.clone(), + expected_invalidated_nodes: invalidated_nodes, + } + }) + .collect(); + Fixture { + manifest, + campaign, + campaign_digest, + schedule, + schedule_digest, + plan, + plan_digest, + observations, + } +} + +fn evaluate(fixture: &Fixture) -> FinalGateReport { + evaluate_final_gate( + FinalGateContract { + manifest: &fixture.manifest, + campaign: &fixture.campaign, + schedule: &fixture.schedule, + power_plan: &fixture.plan, + campaign_digest: &fixture.campaign_digest, + schedule_digest: &fixture.schedule_digest, + power_plan_digest: &fixture.plan_digest, + }, + &fixture.observations, + BootstrapConfig { resamples: 2_000, seed: 42 }, + ) +} + +#[test] +fn complete_schedule_bound_holdout_passes_both_routes() { + let fixture = fixture(CorpusMaterialClass::ProductionSealed); + let report = evaluate(&fixture); + assert!(report.passed, "{:#?}", report); + assert!(report.contract_valid); + assert_eq!(report.schedule_digest, fixture.schedule_digest); + assert_eq!(report.power_plan_digest, fixture.plan_digest); + assert_eq!(report.power_plan_artifact_digest, fixture.plan.artifact_digest); + assert_eq!(report.estimator_revision, POWER_PLAN_ESTIMATOR_REVISION); + assert_eq!(report.bootstrap_seed, 42); + assert_eq!(report.bootstrap_resamples, 2_000); + assert!(report.routes.iter().all(|route| { + route.powered + && route.valid_pairs == route.required_pairs.unwrap() + && route.bca_95_ci.unwrap()[1] < 1.0 + && route.validation_failures.is_empty() + })); +} + +#[test] +fn bootstrap_work_is_bounded_before_evaluation() { + let fixture = fixture(CorpusMaterialClass::ProductionSealed); + let report = evaluate_final_gate( + FinalGateContract { + manifest: &fixture.manifest, + campaign: &fixture.campaign, + schedule: &fixture.schedule, + power_plan: &fixture.plan, + campaign_digest: &fixture.campaign_digest, + schedule_digest: &fixture.schedule_digest, + power_plan_digest: &fixture.plan_digest, + }, + &fixture.observations, + BootstrapConfig { resamples: MAX_BOOTSTRAP_RESAMPLES + 1, seed: 42 }, + ); + assert!(!report.contract_valid); + assert!(report.validation_failures.iter().any(|failure| failure.contains("bootstrap"))); + assert!(report.routes.iter().all(|route| route.bca_95_ci.is_none())); +} + +#[test] +fn synthetic_plan_and_calibration_leakage_are_ineligible() { + let synthetic = fixture(CorpusMaterialClass::Synthetic); + let report = evaluate(&synthetic); + assert!(!report.passed); + assert!(report.validation_failures.iter().any(|failure| failure.contains("synthetic"))); + + let mut production = fixture(CorpusMaterialClass::ProductionSealed); + let entry = production.schedule.entries[0].clone(); + production.observations.push(FinalObservation { + corpus_digest: corpus_digest(&production.manifest), + schedule_digest: production.schedule_digest.clone(), + power_plan_digest: production.plan_digest.clone(), + task_id: entry.task_id, + route: entry.route, + split: entry.split, + arm: entry.arm, + repetition: entry.repetition, + pair_seed: entry.pair_seed, + quality_passed: true, + stale_hit: false, + infrastructure_failure: None, + total_cost_microcredits: Some(1_000), + worker_spawns: 1, + main_discovery_on_covered_scope: 0, + recomputed_nodes: Vec::new(), + expected_invalidated_nodes: Vec::new(), + }); + let report = evaluate(&production); + assert!(!report.passed); + assert!(report.validation_failures.iter().any(|failure| failure.contains("cannot enter"))); +} + +#[test] +fn final_gate_rejects_missing_duplicate_extra_and_stale_identities() { + let base = fixture(CorpusMaterialClass::ProductionSealed); + let mut cases = Vec::<(&str, Fixture)>::new(); + + let mut missing = fixture(CorpusMaterialClass::ProductionSealed); + missing.observations.pop(); + cases.push(("missing", missing)); + + let mut duplicate = fixture(CorpusMaterialClass::ProductionSealed); + duplicate.observations.push(duplicate.observations[0].clone()); + cases.push(("duplicate", duplicate)); + + let mut seed = fixture(CorpusMaterialClass::ProductionSealed); + seed.observations[0].pair_seed += 1; + cases.push(("mismatched seed", seed)); + + let mut split = fixture(CorpusMaterialClass::ProductionSealed); + split.observations[0].split = CorpusSplit::Calibration; + cases.push(("cross split", split)); + + let mut route = fixture(CorpusMaterialClass::ProductionSealed); + route.observations[0].route = BenchmarkRoute::TraceStateFlow; + cases.push(("cross route", route)); + + let mut stale = fixture(CorpusMaterialClass::ProductionSealed); + stale.observations[0].schedule_digest = digest('9'); + cases.push(("stale digest", stale)); + + let mut extra = fixture(CorpusMaterialClass::ProductionSealed); + let mut observation = extra.observations[0].clone(); + observation.repetition = 999; + extra.observations.push(observation); + cases.push(("extra repetition", extra)); + + for (name, fixture) in cases { + let report = evaluate(&fixture); + assert!(!report.passed, "case `{name}` unexpectedly passed"); + assert!( + !report.validation_failures.is_empty() + || report.routes.iter().any(|route| !route.validation_failures.is_empty()), + "case `{name}` did not record its failure" + ); + } + assert!(evaluate(&base).passed); +} + +#[test] +fn failed_evidence_is_never_filtered_into_a_favorable_subset() { + let mut cases = Vec::<(&str, Fixture)>::new(); + + let mut infrastructure = fixture(CorpusMaterialClass::ProductionSealed); + infrastructure.observations[0].infrastructure_failure = Some("transport".to_owned()); + cases.push(("infrastructure", infrastructure)); + + let mut quality = fixture(CorpusMaterialClass::ProductionSealed); + quality.observations[0].quality_passed = false; + cases.push(("quality", quality)); + + let mut stale = fixture(CorpusMaterialClass::ProductionSealed); + stale.observations[0].stale_hit = true; + cases.push(("stale", stale)); + + let mut cost = fixture(CorpusMaterialClass::ProductionSealed); + cost.observations[0].total_cost_microcredits = Some(0); + cases.push(("non-positive cost", cost)); + + let mut worker = fixture(CorpusMaterialClass::ProductionSealed); + let exact = worker.observations.iter_mut().find(|item| item.arm == FinalArm::ExactHit).unwrap(); + exact.worker_spawns = 1; + cases.push(("worker count", worker)); + + let mut invalidation = fixture(CorpusMaterialClass::ProductionSealed); + let partial = + invalidation.observations.iter_mut().find(|item| item.arm == FinalArm::PartialHit).unwrap(); + partial.recomputed_nodes = vec!["unrelated".to_owned()]; + cases.push(("invalidated nodes", invalidation)); + + let mut discovery = fixture(CorpusMaterialClass::ProductionSealed); + discovery.observations[0].main_discovery_on_covered_scope = 1; + cases.push(("main discovery", discovery)); + + for (name, fixture) in cases { + let report = evaluate(&fixture); + assert!(!report.passed, "case `{name}` unexpectedly passed"); + assert!( + report.routes.iter().any(|route| { + !route.validation_failures.is_empty() + && route.paired_cost_ratio.is_none() + && route.bca_95_ci.is_none() + }), + "case `{name}` retained favorable statistics" + ); + } +} + +#[test] +fn holdout_values_cannot_change_the_frozen_power_plan() { + let mut fixture = fixture(CorpusMaterialClass::ProductionSealed); + let required = fixture.plan.routes.iter().map(|route| route.required_pairs).collect::>(); + fixture + .observations + .iter_mut() + .filter(|item| item.arm == FinalArm::NeedleMiss) + .for_each(|item| item.total_cost_microcredits = Some(1_100)); + let report = evaluate(&fixture); + assert_eq!( + report.routes.iter().map(|route| route.required_pairs.unwrap() as u32).collect::>(), + required + ); + assert!(!report.passed); + assert!(report.routes.iter().all(|route| !route.powered || route.bca_95_ci.unwrap()[1] >= 1.0)); +} + +#[test] +fn forged_power_plan_content_or_schedule_count_fails_before_statistics() { + let mut mismatched_campaign = fixture(CorpusMaterialClass::ProductionSealed); + mismatched_campaign.campaign.budget_reserve.main_turn_microcredits += 1; + let report = evaluate(&mismatched_campaign); + assert!(!report.contract_valid); + assert!( + report.validation_failures.iter().any(|failure| failure.contains("campaign commitment")) + ); + + let mut forged = fixture(CorpusMaterialClass::ProductionSealed); + forged.plan.routes[0].required_pairs += 1; + let report = evaluate(&forged); + assert!(!report.contract_valid); + assert!( + report.validation_failures.iter().any(|failure| { + failure.contains("artifact digest") || failure.contains("pair count") + }) + ); + + let mut stale_schedule = fixture(CorpusMaterialClass::ProductionSealed); + stale_schedule.schedule.entries.pop(); + let report = evaluate(&stale_schedule); + assert!(!report.contract_valid); + assert!(report.validation_failures.iter().any(|failure| { + failure.contains("pair count") + || failure.contains("missing") + || failure.contains("complete arm set") + })); +} + +#[test] +fn power_plan_shape_rejects_changed_estimator_contract() { + let mut fixture = fixture(CorpusMaterialClass::ProductionSealed); + fixture.plan.estimator_revision = "changed".to_owned(); + fixture.plan.alpha_basis_points = 499; + fixture.plan.routes[0].pair_key = "list-order".to_owned(); + fixture.plan = fixture.plan.seal(); + let failures = fixture.plan.validate(&fixture.manifest); + assert!(failures.iter().any(|failure| failure.contains("estimator"))); + assert!(failures.iter().any(|failure| failure.contains("alpha"))); + assert!(failures.iter().any(|failure| failure.contains("bounded values"))); + assert_eq!(fixture.plan.schema, POWER_PLAN_SCHEMA); + assert_eq!( + POWER_PLAN_PAIR_KEY, + "corpus_digest:campaign_commitment:task_id:route:split:repetition:pair_seed" + ); +} diff --git a/crates/needle-bench/src/lib.rs b/crates/needle-bench/src/lib.rs index 8d61f1f..6c48bc5 100644 --- a/crates/needle-bench/src/lib.rs +++ b/crates/needle-bench/src/lib.rs @@ -5,6 +5,7 @@ mod calibration_replay; mod corpus; mod final_gate; mod minimal_pilot; +mod power_plan; mod product; mod schedule; mod sealed_oracle; @@ -15,6 +16,7 @@ pub use calibration_replay::*; pub use corpus::*; pub use final_gate::*; pub use minimal_pilot::*; +pub use power_plan::*; pub use product::*; pub use schedule::*; pub use sealed_oracle::*; diff --git a/crates/needle-bench/src/power_plan.rs b/crates/needle-bench/src/power_plan.rs new file mode 100644 index 0000000..f54a4db --- /dev/null +++ b/crates/needle-bench/src/power_plan.rs @@ -0,0 +1,325 @@ +use crate::{ + BenchmarkRoute, CorpusMaterialClass, CorpusSplit, FinalArm, FrozenCorpusManifest, + MAX_POWER_PLAN_PAIRS, MAX_SCHEDULE_ENTRIES, MultiTaskCampaign, POWER_PLAN_ESTIMATOR_REVISION, + POWER_PLAN_PAIR_KEY, POWER_PLAN_SCHEMA, PowerPlan, PowerRoutePlan, campaign_commitment, + corpus_digest, digest_json, +}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +pub const POWER_PLANNING_REPORT_SCHEMA: &str = "needle.power-planning-report/1"; +pub const POWER_PLAN_ALPHA_BASIS_POINTS: u16 = 500; +pub const POWER_PLAN_TARGET_POWER_BASIS_POINTS: u16 = 9_000; +pub const MAX_CALIBRATION_INPUT_BYTES: usize = 16 * 1024 * 1024; +const MAX_FAILURE_DETAIL_BYTES: usize = 512; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CalibrationObservation { + pub corpus_digest: String, + pub campaign_commitment: String, + pub task_id: String, + pub route: BenchmarkRoute, + pub split: CorpusSplit, + pub arm: FinalArm, + pub repetition: u32, + pub pair_seed: u64, + pub quality_passed: bool, + pub infrastructure_failure: Option, + pub total_cost_microcredits: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PowerPlanningReport { + pub schema: String, + pub manifest_digest: String, + pub campaign_commitment: String, + pub calibration_input_digest: String, + pub estimator_revision: String, + pub alpha_basis_points: u16, + pub target_power_basis_points: u16, + pub plan: Option, + pub failures: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct PowerEstimate { + pub observed_log_ratio_mean: f64, + pub observed_log_ratio_stddev: f64, + pub required_pairs_per_route: usize, + pub power: f64, + pub one_sided_alpha: f64, +} + +pub fn estimate_required_pairs(calibration_ratios: &[f64]) -> Option { + if calibration_ratios.len() < 2 + || calibration_ratios.iter().any(|ratio| !ratio.is_finite() || *ratio <= 0.0) + { + return None; + } + let logs = calibration_ratios.iter().map(|ratio| ratio.ln()).collect::>(); + let mean = average(&logs); + if mean >= 0.0 { + return None; + } + let variance = + logs.iter().map(|value| (value - mean).powi(2)).sum::() / (logs.len() - 1) as f64; + let stddev = variance.sqrt(); + if !stddev.is_finite() || stddev == 0.0 { + return None; + } + let required = required_pairs_for_moments(mean, stddev)?; + Some(PowerEstimate { + observed_log_ratio_mean: mean, + observed_log_ratio_stddev: stddev, + required_pairs_per_route: required, + power: 0.90, + one_sided_alpha: 0.05, + }) +} + +pub fn required_pairs_for_moments(mean: f64, stddev: f64) -> Option { + if !mean.is_finite() || mean >= 0.0 || !stddev.is_finite() || stddev <= 0.0 { + return None; + } + let required = (((1.644_853_626_951_472_2 + 1.281_551_565_544_600_4) * stddev / mean.abs()) + .powi(2)) + .ceil() + .max(3.0); + (required.is_finite() && required <= usize::MAX as f64).then_some(required as usize) +} + +pub(crate) fn validate_power_campaign(campaign: &MultiTaskCampaign) -> Vec { + if campaign.schema != "needle.multi-task-campaign/2" + || campaign.paid_arms != [FinalArm::FrontierDirect, FinalArm::NeedleMiss] + || campaign.offline_cache_arms + != [ + FinalArm::ExactHit, + FinalArm::PartialHit, + FinalArm::IrrelevantMutation, + FinalArm::RelevantMutation, + ] + || campaign.deferred_diagnostic_arms != [FinalArm::NativeSubagent, FinalArm::Escalation] + || !campaign.task_ids.is_empty() + || campaign.repetitions_per_task != 0 + || campaign.automatic_retries + || campaign.statistical_claim + || campaign.bootstrap_resamples < 1_000 + || campaign.one_sided_alpha_basis_points != POWER_PLAN_ALPHA_BASIS_POINTS + || campaign.target_power_basis_points != POWER_PLAN_TARGET_POWER_BASIS_POINTS + { + vec!["power planning campaign contract is invalid".to_owned()] + } else { + Vec::new() + } +} + +pub fn plan_power( + manifest: &FrozenCorpusManifest, + campaign: &MultiTaskCampaign, + observations: &[CalibrationObservation], +) -> PowerPlanningReport { + let campaign_commitment = campaign_commitment(campaign); + let manifest_digest = corpus_digest(manifest); + let mut canonical = observations.to_vec(); + canonical.sort_by_key(observation_key); + let calibration_input_digest = digest_json(&canonical); + let mut failures = Vec::new(); + + if manifest.schema != "needle.frozen-corpus/4" { + failures.push("power planning requires a frozen corpus v4 manifest".to_owned()); + } + failures.extend(validate_power_campaign(campaign)); + if observations.len() > MAX_SCHEDULE_ENTRIES { + failures.push("calibration observation count exceeds bounded maximum".to_owned()); + } + + let calibration_tasks = manifest + .tasks + .iter() + .filter(|task| task.split == CorpusSplit::Calibration) + .map(|task| (task.id.as_str(), task)) + .collect::>(); + let material = + calibration_tasks.values().map(|task| task.material_class).collect::>(); + let synthetic = material == BTreeSet::from([CorpusMaterialClass::Synthetic]); + if material.len() != 1 || material.contains(&CorpusMaterialClass::Legacy) { + failures.push("calibration tasks must use one non-legacy material class".to_owned()); + } + + let mut observation_keys = BTreeSet::new(); + let mut observed_tasks = BTreeSet::new(); + let mut pairs = + BTreeMap::<(String, BenchmarkRoute, u32, u64), (Option, Option)>::new(); + for observation in &canonical { + if observation.corpus_digest != manifest_digest + || observation.campaign_commitment != campaign_commitment + { + failures.push(format!( + "calibration observation `{}` has stale corpus or campaign identity", + observation.task_id + )); + } + let Some(task) = calibration_tasks.get(observation.task_id.as_str()) else { + failures.push(format!( + "calibration observation references unknown task `{}`", + observation.task_id + )); + continue; + }; + observed_tasks.insert(observation.task_id.as_str()); + if observation.split != CorpusSplit::Calibration + || task.route != observation.route + || task.split != observation.split + { + failures.push(format!( + "calibration observation metadata differs for task `{}`", + observation.task_id + )); + } + if !matches!(observation.arm, FinalArm::FrontierDirect | FinalArm::NeedleMiss) { + failures.push(format!( + "calibration observation `{}` uses a non-economic arm", + observation.task_id + )); + continue; + } + if observation.infrastructure_failure.is_some() + || !observation.quality_passed + || observation.total_cost_microcredits.is_none_or(|cost| cost == 0) + || observation + .infrastructure_failure + .as_ref() + .is_some_and(|detail| detail.len() > MAX_FAILURE_DETAIL_BYTES) + { + failures.push(format!( + "calibration observation `{}` is failed, low-quality, or has invalid cost", + observation.task_id + )); + } + if !observation_keys.insert(observation_key(observation)) { + failures.push(format!( + "duplicate calibration observation identity for task `{}`", + observation.task_id + )); + continue; + } + let pair = pairs + .entry(( + observation.task_id.clone(), + observation.route, + observation.repetition, + observation.pair_seed, + )) + .or_default(); + match observation.arm { + FinalArm::FrontierDirect => pair.0 = observation.total_cost_microcredits, + FinalArm::NeedleMiss => pair.1 = observation.total_cost_microcredits, + _ => unreachable!(), + } + } + for task_id in calibration_tasks.keys() { + if !observed_tasks.contains(task_id) { + failures.push(format!("calibration task `{task_id}` has no observations")); + } + } + + let mut ratios = BTreeMap::>::new(); + for ((task_id, route, repetition, pair_seed), (baseline, treatment)) in pairs { + match (baseline, treatment) { + (Some(baseline), Some(treatment)) if baseline > 0 && treatment > 0 => { + ratios.entry(route).or_default().push(treatment as f64 / baseline as f64); + } + _ => failures.push(format!( + "calibration pair `{task_id}` repetition {repetition} seed {pair_seed} is incomplete" + )), + } + } + + let mut routes = Vec::new(); + for route in [BenchmarkRoute::LocateImplementation, BenchmarkRoute::TraceStateFlow] { + let route_ratios = ratios.remove(&route).unwrap_or_default(); + let Some(estimate) = estimate_required_pairs(&route_ratios) else { + failures.push(format!( + "route {route:?} has insufficient, non-beneficial, or zero-variance calibration" + )); + continue; + }; + if estimate.required_pairs_per_route > MAX_POWER_PLAN_PAIRS as usize { + failures.push(format!("route {route:?} exceeds the bounded holdout pair maximum")); + continue; + } + routes.push(PowerRoutePlan { + route, + baseline_arm: FinalArm::FrontierDirect, + treatment_arm: FinalArm::NeedleMiss, + pair_key: POWER_PLAN_PAIR_KEY.to_owned(), + observed_log_ratio_mean: estimate.observed_log_ratio_mean, + observed_log_ratio_stddev: estimate.observed_log_ratio_stddev, + required_pairs: estimate.required_pairs_per_route as u32, + }); + } + + let plan = if failures.is_empty() { + let plan = PowerPlan { + schema: POWER_PLAN_SCHEMA.to_owned(), + plan_id: format!("power-plan-{}", &calibration_input_digest[3..19]), + manifest_digest: manifest_digest.clone(), + campaign_commitment: campaign_commitment.clone(), + calibration_input_digest: calibration_input_digest.clone(), + estimator_revision: POWER_PLAN_ESTIMATOR_REVISION.to_owned(), + alpha_basis_points: POWER_PLAN_ALPHA_BASIS_POINTS, + target_power_basis_points: POWER_PLAN_TARGET_POWER_BASIS_POINTS, + routes, + validated: true, + synthetic, + artifact_digest: String::new(), + } + .seal(); + let validation = plan.validate(manifest); + if validation.is_empty() { + Some(plan) + } else { + failures.extend(validation); + None + } + } else { + None + }; + + PowerPlanningReport { + schema: POWER_PLANNING_REPORT_SCHEMA.to_owned(), + manifest_digest, + campaign_commitment, + calibration_input_digest, + estimator_revision: POWER_PLAN_ESTIMATOR_REVISION.to_owned(), + alpha_basis_points: POWER_PLAN_ALPHA_BASIS_POINTS, + target_power_basis_points: POWER_PLAN_TARGET_POWER_BASIS_POINTS, + plan, + failures, + } +} + +fn observation_key( + observation: &CalibrationObservation, +) -> (String, String, String, BenchmarkRoute, CorpusSplit, u32, u64, FinalArm) { + ( + observation.corpus_digest.clone(), + observation.campaign_commitment.clone(), + observation.task_id.clone(), + observation.route, + observation.split, + observation.repetition, + observation.pair_seed, + observation.arm, + ) +} + +fn average(values: &[f64]) -> f64 { + values.iter().sum::() / values.len() as f64 +} + +#[cfg(test)] +#[path = "power_plan/tests.rs"] +mod tests; diff --git a/crates/needle-bench/src/power_plan/tests.rs b/crates/needle-bench/src/power_plan/tests.rs new file mode 100644 index 0000000..9b53c7a --- /dev/null +++ b/crates/needle-bench/src/power_plan/tests.rs @@ -0,0 +1,261 @@ +use super::*; +use crate::{CampaignBudgetReserve, CorpusTask, FocusedTestPolicyRef}; + +fn digest(byte: char) -> String { + format!("b3:{}", byte.to_string().repeat(64)) +} + +fn task( + id: &str, + route: BenchmarkRoute, + split: CorpusSplit, + material_class: CorpusMaterialClass, +) -> CorpusTask { + CorpusTask { + id: id.to_owned(), + route, + split, + repository_url: "https://example.invalid/repository.git".to_owned(), + repository_sha: "0123456789abcdef0123456789abcdef01234567".to_owned(), + prompt: "A bounded benchmark task prompt with no embedded answer material.".to_owned(), + material_class, + focused_test_policy: FocusedTestPolicyRef { + identity: format!("policy-{id}"), + commitment: digest('1'), + }, + oracle_schema: "needle.sealed-oracle/1".to_owned(), + oracle_digest: digest('2'), + oracle_path: String::new(), + test_identifier: String::new(), + focused_command: Vec::new(), + } +} + +fn manifest(material_class: CorpusMaterialClass) -> FrozenCorpusManifest { + FrozenCorpusManifest { + schema: "needle.frozen-corpus/4".to_owned(), + frozen_unix_ms: 1, + arms: FinalArm::ALL.to_vec(), + cost_model_path: "cost-model.json".to_owned(), + cost_model_digest: digest('3'), + next_pilot_path: "pilot.json".to_owned(), + next_pilot_digest: digest('4'), + campaign_path: Some("campaign.json".to_owned()), + campaign_digest: Some(digest('5')), + schedule_path: Some("schedule.json".to_owned()), + schedule_digest: Some(digest('6')), + power_plan_path: Some("power-plan.json".to_owned()), + power_plan_digest: Some(digest('7')), + sealed_bundle_schema: Some("needle.sealed-oracle-index/1".to_owned()), + sealed_bundle_digest: Some(digest('8')), + tasks: vec![ + task( + "cal-locate", + BenchmarkRoute::LocateImplementation, + CorpusSplit::Calibration, + material_class, + ), + task( + "cal-trace", + BenchmarkRoute::TraceStateFlow, + CorpusSplit::Calibration, + material_class, + ), + task( + "hold-locate", + BenchmarkRoute::LocateImplementation, + CorpusSplit::Holdout, + material_class, + ), + task( + "hold-trace", + BenchmarkRoute::TraceStateFlow, + CorpusSplit::Holdout, + material_class, + ), + ], + } +} + +fn campaign() -> MultiTaskCampaign { + MultiTaskCampaign { + schema: "needle.multi-task-campaign/2".to_owned(), + schedule_digest: digest('6'), + task_ids: Vec::new(), + paid_arms: vec![FinalArm::FrontierDirect, FinalArm::NeedleMiss], + offline_cache_arms: vec![ + FinalArm::ExactHit, + FinalArm::PartialHit, + FinalArm::IrrelevantMutation, + FinalArm::RelevantMutation, + ], + deferred_diagnostic_arms: vec![FinalArm::NativeSubagent, FinalArm::Escalation], + repetitions_per_task: 0, + automatic_retries: false, + statistical_claim: false, + bootstrap_resamples: 10_000, + one_sided_alpha_basis_points: POWER_PLAN_ALPHA_BASIS_POINTS, + target_power_basis_points: POWER_PLAN_TARGET_POWER_BASIS_POINTS, + budget_reserve: CampaignBudgetReserve { + main_turn_microcredits: 1, + extra_main_turns_per_needle_observation: 1, + worker_microcredits: 1, + extra_workers_per_needle_observation: 0, + evidence: vec!["synthetic".to_owned()], + }, + } +} + +fn observations( + manifest: &FrozenCorpusManifest, + campaign: &MultiTaskCampaign, +) -> Vec { + let corpus = corpus_digest(manifest); + let campaign = campaign_commitment(campaign); + let mut observations = Vec::new(); + for (task_id, route) in [ + ("cal-locate", BenchmarkRoute::LocateImplementation), + ("cal-trace", BenchmarkRoute::TraceStateFlow), + ] { + for (repetition, treatment) in [(0, 600), (1, 700)] { + for (arm, cost) in + [(FinalArm::FrontierDirect, 1_000), (FinalArm::NeedleMiss, treatment)] + { + observations.push(CalibrationObservation { + corpus_digest: corpus.clone(), + campaign_commitment: campaign.clone(), + task_id: task_id.to_owned(), + route, + split: CorpusSplit::Calibration, + arm, + repetition, + pair_seed: 100 + repetition as u64, + quality_passed: true, + infrastructure_failure: None, + total_cost_microcredits: Some(cost), + }); + } + } + } + observations +} + +#[test] +fn planning_is_deterministic_and_digest_bound() { + let manifest = manifest(CorpusMaterialClass::ProductionSealed); + let campaign = campaign(); + let observations = observations(&manifest, &campaign); + let first = plan_power(&manifest, &campaign, &observations); + let mut reordered = observations.clone(); + reordered.reverse(); + let second = plan_power(&manifest, &campaign, &reordered); + assert!(first.failures.is_empty(), "{:?}", first.failures); + assert_eq!(first, second); + let plan = first.plan.unwrap(); + assert!(!plan.synthetic); + assert_eq!(plan.artifact_digest, plan.canonical_digest()); + assert!(plan.routes.iter().all(|route| route.required_pairs >= 3)); + + let mut changed = observations; + changed[0].total_cost_microcredits = Some(1_001); + let changed = plan_power(&manifest, &campaign, &changed); + assert_ne!(changed.calibration_input_digest, second.calibration_input_digest); + assert_ne!(changed.plan.unwrap().artifact_digest, plan.artifact_digest); +} + +#[test] +fn synthetic_material_is_derived_and_cannot_be_presented_as_real() { + let manifest = manifest(CorpusMaterialClass::Synthetic); + let campaign = campaign(); + let report = plan_power(&manifest, &campaign, &observations(&manifest, &campaign)); + let plan = report.plan.unwrap(); + assert!(plan.synthetic); + let mut forged = plan; + forged.synthetic = false; + forged = forged.seal(); + assert!( + forged + .validate(&manifest) + .iter() + .any(|failure| failure.contains("material classification")) + ); +} + +#[test] +fn planning_rejects_changed_campaign_parameters() { + let manifest = manifest(CorpusMaterialClass::ProductionSealed); + let mut campaign = campaign(); + let mut observations = observations(&manifest, &campaign); + campaign.one_sided_alpha_basis_points = 499; + let changed_commitment = campaign_commitment(&campaign); + observations + .iter_mut() + .for_each(|observation| observation.campaign_commitment = changed_commitment.clone()); + let report = plan_power(&manifest, &campaign, &observations); + assert!(report.plan.is_none()); + assert!(report.failures.iter().any(|failure| failure.contains("campaign contract"))); +} + +#[test] +fn planning_fails_closed_for_incomplete_or_contaminated_calibration() { + let manifest = manifest(CorpusMaterialClass::ProductionSealed); + let campaign = campaign(); + let baseline = observations(&manifest, &campaign); + let mut cases = Vec::<(&str, Vec)>::new(); + + let mut missing = baseline.clone(); + missing.pop(); + cases.push(("missing counterpart", missing)); + + let mut duplicate = baseline.clone(); + duplicate.push(duplicate[0].clone()); + cases.push(("duplicate arm", duplicate)); + + let mut mismatched_seed = baseline.clone(); + mismatched_seed[1].pair_seed += 1; + cases.push(("mismatched seed", mismatched_seed)); + + let mut non_positive = baseline.clone(); + non_positive[0].total_cost_microcredits = Some(0); + cases.push(("non-positive cost", non_positive)); + + let mut quality = baseline.clone(); + quality[0].quality_passed = false; + cases.push(("quality failure", quality)); + + let mut infrastructure = baseline.clone(); + infrastructure[0].infrastructure_failure = Some("transport".to_owned()); + cases.push(("infrastructure failure", infrastructure)); + + let mut split = baseline.clone(); + split[0].split = CorpusSplit::Holdout; + cases.push(("split contamination", split)); + + let mut route = baseline.clone(); + route[0].route = BenchmarkRoute::TraceStateFlow; + cases.push(("route contamination", route)); + + let mut stale = baseline.clone(); + stale[0].corpus_digest = digest('9'); + cases.push(("stale corpus", stale)); + + let mut non_beneficial = baseline.clone(); + non_beneficial + .iter_mut() + .filter(|item| item.arm == FinalArm::NeedleMiss) + .for_each(|item| item.total_cost_microcredits = Some(1_100 + item.repetition as u64 * 100)); + cases.push(("non-beneficial effect", non_beneficial)); + + let mut zero_variance = baseline; + zero_variance + .iter_mut() + .filter(|item| item.arm == FinalArm::NeedleMiss) + .for_each(|item| item.total_cost_microcredits = Some(600)); + cases.push(("zero variance", zero_variance)); + + for (name, observations) in cases { + let report = plan_power(&manifest, &campaign, &observations); + assert!(report.plan.is_none(), "case `{name}` unexpectedly produced a plan"); + assert!(!report.failures.is_empty(), "case `{name}` did not explain the failure"); + } +} diff --git a/crates/needle-bench/src/schedule.rs b/crates/needle-bench/src/schedule.rs index 29cbc21..893c63d 100644 --- a/crates/needle-bench/src/schedule.rs +++ b/crates/needle-bench/src/schedule.rs @@ -6,8 +6,11 @@ use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; pub const CORPUS_SCHEDULE_SCHEMA: &str = "needle.corpus-schedule/1"; -pub const POWER_PLAN_SCHEMA: &str = "needle.power-plan/1"; +pub const POWER_PLAN_SCHEMA: &str = "needle.power-plan/2"; pub const ARM_LAUNCH_SCHEMA: &str = "needle.arm-launch/1"; +pub const POWER_PLAN_ESTIMATOR_REVISION: &str = "needle.paired-log-ratio-power/1"; +pub const POWER_PLAN_PAIR_KEY: &str = + "corpus_digest:campaign_commitment:task_id:route:split:repetition:pair_seed"; pub const MAX_POWER_PLAN_PAIRS: u32 = 10_000; pub const MAX_SCHEDULE_ENTRIES: usize = 100_000; pub const MAX_SCHEDULE_BYTES: usize = 4 * 1024 * 1024; @@ -16,12 +19,13 @@ const MAX_PROMPT_BYTES: usize = 4_000; #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ScheduleEntry { +pub struct CorpusScheduleEntry { pub task_id: String, pub route: BenchmarkRoute, pub split: CorpusSplit, pub arm: FinalArm, pub repetition: u32, + pub pair_seed: u64, } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -52,7 +56,7 @@ pub struct PowerPlan { pub schema: String, pub plan_id: String, pub manifest_digest: String, - pub campaign_digest: String, + pub campaign_commitment: String, pub calibration_input_digest: String, pub estimator_revision: String, pub alpha_basis_points: u16, @@ -60,6 +64,7 @@ pub struct PowerPlan { pub routes: Vec, pub validated: bool, pub synthetic: bool, + pub artifact_digest: String, } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -76,6 +81,17 @@ impl PowerPlanReference { } impl PowerPlan { + pub fn canonical_digest(&self) -> String { + let mut canonical = self.clone(); + canonical.artifact_digest.clear(); + digest_json(&canonical) + } + + pub fn seal(mut self) -> Self { + self.artifact_digest = self.canonical_digest(); + self + } + pub fn validate(&self, manifest: &FrozenCorpusManifest) -> Vec { let mut errors = Vec::new(); if self.schema != POWER_PLAN_SCHEMA { @@ -87,13 +103,15 @@ impl PowerPlan { if self.manifest_digest != corpus_digest(manifest) { errors.push("power plan manifest digest differs".to_owned()); } - if !valid_digest(&self.campaign_digest) { - errors.push("power plan campaign digest is invalid".to_owned()); + if !valid_digest(&self.campaign_commitment) { + errors.push("power plan campaign commitment is invalid".to_owned()); } if !valid_digest(&self.calibration_input_digest) { errors.push("power plan calibration-input digest is invalid".to_owned()); + } else if self.plan_id != format!("power-plan-{}", &self.calibration_input_digest[3..19]) { + errors.push("power plan id differs from the calibration-input digest".to_owned()); } - if self.estimator_revision.trim().is_empty() || self.estimator_revision.len() > 128 { + if self.estimator_revision != POWER_PLAN_ESTIMATOR_REVISION { errors.push("power plan estimator revision is invalid".to_owned()); } if self.alpha_basis_points != 500 || self.target_power_basis_points != 9_000 { @@ -113,15 +131,19 @@ impl PowerPlan { } if route.baseline_arm != FinalArm::FrontierDirect || route.treatment_arm != FinalArm::NeedleMiss - || route.pair_key != "task_id:repetition" + || route.pair_key != POWER_PLAN_PAIR_KEY || !route.observed_log_ratio_mean.is_finite() || route.observed_log_ratio_mean >= 0.0 || route.observed_log_ratio_mean < -100.0 || !route.observed_log_ratio_stddev.is_finite() || route.observed_log_ratio_stddev <= 0.0 || route.observed_log_ratio_stddev > 100.0 - || route.required_pairs == 0 + || route.required_pairs < 3 || route.required_pairs > MAX_POWER_PLAN_PAIRS + || crate::required_pairs_for_moments( + route.observed_log_ratio_mean, + route.observed_log_ratio_stddev, + ) != Some(route.required_pairs as usize) { errors .push(format!("power plan route {:?} has invalid bounded values", route.route)); @@ -130,6 +152,25 @@ impl PowerPlan { if !self.validated { errors.push("power plan has not been validated".to_owned()); } + let calibration_material = manifest + .tasks + .iter() + .filter(|task| task.split == CorpusSplit::Calibration) + .map(|task| task.material_class) + .collect::>(); + let expected_synthetic = + calibration_material == BTreeSet::from([CorpusMaterialClass::Synthetic]); + if calibration_material.iter().any(|material| *material == CorpusMaterialClass::Legacy) + || calibration_material.len() != 1 + || self.synthetic != expected_synthetic + { + errors.push( + "power plan material classification differs from calibration tasks".to_owned(), + ); + } + if self.artifact_digest != self.canonical_digest() { + errors.push("power plan artifact digest differs from canonical content".to_owned()); + } errors } @@ -145,7 +186,7 @@ pub struct CorpusSchedule { pub manifest_digest: String, pub power_plan_digest: String, pub automatic_retries: bool, - pub entries: Vec, + pub entries: Vec, } impl CorpusSchedule { @@ -155,7 +196,7 @@ impl CorpusSchedule { power_plan: &PowerPlan, raw_power_plan_digest: &str, ) -> Vec { - let mut errors = Vec::new(); + let mut errors = power_plan.validate(manifest); if self.schema != CORPUS_SCHEDULE_SCHEMA { errors.push("corpus schedule schema is unsupported".to_owned()); } @@ -200,6 +241,7 @@ pub struct ArmLaunch { pub prompt: String, pub arm: FinalArm, pub repetition: u32, + pub pair_seed: u64, pub focused_test_policy_identity: String, pub focused_test_policy_commitment: String, } @@ -209,6 +251,7 @@ impl ArmLaunch { task: &CorpusTask, arm: FinalArm, repetition: u32, + pair_seed: u64, manifest_digest: &str, schedule_digest: &str, power_plan_digest: &str, @@ -245,6 +288,7 @@ impl ArmLaunch { prompt: task.prompt.clone(), arm, repetition, + pair_seed, focused_test_policy_identity: task.focused_test_policy.identity.clone(), focused_test_policy_commitment: task.focused_test_policy.commitment.clone(), }) @@ -294,6 +338,7 @@ pub fn build_launch_plan( task, entry.arm, entry.repetition, + entry.pair_seed, &manifest_digest, schedule_digest, power_plan_digest, @@ -310,9 +355,10 @@ fn validate_entries_in_manifest_order( errors: &mut Vec, ) { let mut offset = 0usize; - let mut holdout_pairs = BTreeMap::>::new(); + let mut holdout_pairs = BTreeMap::>::new(); let calibration_arms = [FinalArm::FrontierDirect, FinalArm::NeedleMiss]; for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Calibration) { + let mut pair_seed = None; for arm in calibration_arms { let Some(entry) = schedule.entries.get(offset) else { errors.push("schedule is missing a calibration entry".to_owned()); @@ -328,12 +374,19 @@ fn validate_entries_in_manifest_order( "schedule calibration entries differ from canonical task order".to_owned(), ); } + match pair_seed { + Some(expected) if expected != entry.pair_seed => { + errors.push("schedule calibration pair seeds differ between arms".to_owned()); + } + None => pair_seed = Some(entry.pair_seed), + _ => {} + } offset = offset.saturating_add(1); } } for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Holdout) { let mut task_pairs = BTreeSet::new(); - let mut pair_counts = BTreeMap::<(String, u32), usize>::new(); + let mut pair_counts = BTreeMap::<(String, u32, u64), usize>::new(); let task_start = offset; while let Some(entry) = schedule.entries.get(offset) { if entry.task_id != task.id { @@ -342,7 +395,7 @@ fn validate_entries_in_manifest_order( if entry.route != task.route || entry.split != task.split { errors.push("schedule holdout metadata differs from manifest task".to_owned()); } - let pair = (entry.task_id.clone(), entry.repetition); + let pair = (entry.task_id.clone(), entry.repetition, entry.pair_seed); task_pairs.insert(pair.clone()); *pair_counts.entry(pair.clone()).or_default() += 1; holdout_pairs.entry(task.route).or_default().insert(pair); @@ -356,6 +409,10 @@ fn validate_entries_in_manifest_order( "schedule holdout repetitions are not in canonical block order".to_owned(), ); } + let block_start = offset - within_pair; + if schedule.entries[block_start].pair_seed != entry.pair_seed { + errors.push("schedule holdout pair seeds differ between arms".to_owned()); + } offset = offset.saturating_add(1); } if task_pairs.is_empty() { @@ -363,7 +420,7 @@ fn validate_entries_in_manifest_order( continue; } let repetitions = - task_pairs.iter().map(|(_, repetition)| *repetition).collect::>(); + task_pairs.iter().map(|(_, repetition, _)| *repetition).collect::>(); if repetitions .iter() .copied() @@ -463,12 +520,13 @@ mod tests { fn plan(manifest: &FrozenCorpusManifest) -> PowerPlan { PowerPlan { schema: POWER_PLAN_SCHEMA.to_owned(), - plan_id: "synthetic".to_owned(), + plan_id: "power-plan-0123456789abcdef".to_owned(), manifest_digest: corpus_digest(manifest), - campaign_digest: manifest.campaign_digest.clone().unwrap(), + campaign_commitment: + "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), calibration_input_digest: "b3:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".to_owned(), - estimator_revision: "issue-7-structural-v1".to_owned(), + estimator_revision: POWER_PLAN_ESTIMATOR_REVISION.to_owned(), alpha_basis_points: 500, target_power_basis_points: 9_000, routes: vec![ @@ -476,48 +534,55 @@ mod tests { route: BenchmarkRoute::LocateImplementation, baseline_arm: FinalArm::FrontierDirect, treatment_arm: FinalArm::NeedleMiss, - pair_key: "task_id:repetition".to_owned(), + pair_key: POWER_PLAN_PAIR_KEY.to_owned(), observed_log_ratio_mean: -0.4, observed_log_ratio_stddev: 0.2, - required_pairs: 1, + required_pairs: 3, }, PowerRoutePlan { route: BenchmarkRoute::TraceStateFlow, baseline_arm: FinalArm::FrontierDirect, treatment_arm: FinalArm::NeedleMiss, - pair_key: "task_id:repetition".to_owned(), + pair_key: POWER_PLAN_PAIR_KEY.to_owned(), observed_log_ratio_mean: -0.4, observed_log_ratio_stddev: 0.2, - required_pairs: 1, + required_pairs: 3, }, ], validated: true, synthetic: true, + artifact_digest: String::new(), } + .seal() } fn schedule(manifest: &FrozenCorpusManifest, plan: &PowerPlan) -> CorpusSchedule { let mut entries = Vec::new(); for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Calibration) { for arm in [FinalArm::FrontierDirect, FinalArm::NeedleMiss] { - entries.push(ScheduleEntry { + entries.push(CorpusScheduleEntry { task_id: task.id.clone(), route: task.route, split: task.split, arm, repetition: 0, + pair_seed: 1, }); } } for task in manifest.tasks.iter().filter(|task| task.split == CorpusSplit::Holdout) { - for arm in FinalArm::ALL { - entries.push(ScheduleEntry { - task_id: task.id.clone(), - route: task.route, - split: task.split, - arm, - repetition: 0, - }); + let required = plan.required_pairs(task.route).unwrap(); + for repetition in 0..required { + for arm in FinalArm::ALL { + entries.push(CorpusScheduleEntry { + task_id: task.id.clone(), + route: task.route, + split: task.split, + arm, + repetition, + pair_seed: 100 + repetition as u64, + }); + } } } let bytes = serde_json::to_vec(&plan).unwrap(); @@ -572,25 +637,19 @@ mod tests { #[test] fn schedule_rejects_complete_holdout_blocks_in_reversed_repetition_order() { let manifest = manifest(); - let mut plan = plan(&manifest); - for route in &mut plan.routes { - route.required_pairs = 2; - } + let plan = plan(&manifest); let canonical = schedule(&manifest, &plan); let mut entries = canonical.entries[..4].to_vec(); for task_id in ["hold-locate", "hold-trace"] { - let block = canonical + let task_entries = canonical .entries .iter() .filter(|entry| entry.task_id == task_id) .cloned() .collect::>(); - let mut repetition_one = block.clone(); - repetition_one.iter_mut().for_each(|entry| entry.repetition = 1); - let mut repetition_zero = block; - repetition_zero.iter_mut().for_each(|entry| entry.repetition = 0); - entries.extend(repetition_one); - entries.extend(repetition_zero); + entries.extend_from_slice(&task_entries[FinalArm::ALL.len()..FinalArm::ALL.len() * 2]); + entries.extend_from_slice(&task_entries[..FinalArm::ALL.len()]); + entries.extend_from_slice(&task_entries[FinalArm::ALL.len() * 2..]); } let mut reversed = canonical; reversed.entries = entries; diff --git a/crates/needle-bench/tests/publication_surface.rs b/crates/needle-bench/tests/publication_surface.rs index c92bfc6..7247c50 100644 --- a/crates/needle-bench/tests/publication_surface.rs +++ b/crates/needle-bench/tests/publication_surface.rs @@ -1,6 +1,8 @@ use needle_bench::{ - ArmLaunch, CorpusSchedule, FrozenCorpusManifest, PowerPlan, SealedOracleIndex, - build_launch_plan, corpus_digest, raw_digest, validate_sealed_bundle, + ArmLaunch, BootstrapConfig, CalibrationObservation, CorpusSchedule, FinalGateContract, + FinalObservation, FrozenCorpusManifest, MultiTaskCampaign, PowerPlan, SealedOracleIndex, + build_launch_plan, campaign_commitment, corpus_digest, evaluate_final_gate, plan_power, + raw_digest, validate_sealed_bundle, }; use serde_json::Value; use std::collections::BTreeSet; @@ -190,6 +192,19 @@ fn public_v4_fixture_is_answer_free_and_exactly_committed() { serde_json::from_slice(&fs::read(root.join("schedule.json")).unwrap()).unwrap(); let plan: PowerPlan = serde_json::from_slice(&fs::read(root.join("power-plan.json")).unwrap()).unwrap(); + let campaign: MultiTaskCampaign = + serde_json::from_slice(&fs::read(root.join("campaign.json")).unwrap()).unwrap(); + let campaign_digest = raw_digest(&fs::read(root.join("campaign.json")).unwrap()); + let calibration = fs::read_to_string(root.join("synthetic-calibration-observations.jsonl")) + .unwrap() + .lines() + .map(serde_json::from_str::) + .collect::, _>>() + .unwrap(); + assert_eq!(plan.campaign_commitment, campaign_commitment(&campaign)); + let reproduced = plan_power(&manifest, &campaign, &calibration); + assert!(reproduced.failures.is_empty(), "{:?}", reproduced.failures); + assert_eq!(reproduced.plan.as_ref(), Some(&plan)); let plan_digest = raw_digest(&fs::read(root.join("power-plan.json")).unwrap()); let schedule_digest = raw_digest(&fs::read(root.join("schedule.json")).unwrap()); assert_eq!(plan.manifest_digest, corpus_digest(&manifest)); @@ -199,6 +214,33 @@ fn public_v4_fixture_is_answer_free_and_exactly_committed() { build_launch_plan(&manifest, &schedule, &plan, &schedule_digest, &plan_digest).unwrap(); assert_eq!(launches.len(), schedule.entries.len()); assert!(launches.iter().all(ArmLaunch::serialization_is_bounded)); + let final_observations = fs::read_to_string(root.join("synthetic-final-observations.jsonl")) + .unwrap() + .lines() + .map(serde_json::from_str::) + .collect::, _>>() + .unwrap(); + let final_report = evaluate_final_gate( + FinalGateContract { + manifest: &manifest, + campaign: &campaign, + schedule: &schedule, + power_plan: &plan, + campaign_digest: &campaign_digest, + schedule_digest: &schedule_digest, + power_plan_digest: &plan_digest, + }, + &final_observations, + BootstrapConfig { resamples: 2_000, seed: 42 }, + ); + assert!(!final_report.passed); + assert!(!final_report.contract_valid); + assert!(final_report.validation_failures.iter().any(|failure| failure.contains("synthetic"))); + assert!(final_report.routes.iter().all(|route| { + route.valid_pairs == route.required_pairs.unwrap_or_default() + && route.paired_cost_ratio.is_none() + && route.bca_95_ci.is_none() + })); let bundle_root = root.join("synthetic-sealed"); let bundle = validate_sealed_bundle( &manifest, diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 30ca20d..1eb34f1 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -45,8 +45,9 @@ Public corpus manifests use `needle.frozen-corpus/4`: they contain only answer-free prompts, source identities, policy commitments, and oracle commitments. Answer bytes and quality/test policy live in an evaluator-owned sealed bundle indexed outside the public manifest. The checked-in router-cache -tasks and power plan are synthetic fixtures and are permanently ineligible for -provider evidence. +tasks, calibration observations, and power plan are synthetic fixtures and are +permanently ineligible for provider evidence. The synthetic calibration JSONL +exists only to reproduce the checked-in plan and exercise the offline protocol. The app provider path remains fail-closed until an isolated executor/broker consumes only the bounded ArmLaunch projection. Evaluator callers must keep @@ -172,6 +173,32 @@ automatic retry. ## Final statistical gate +Power planning and final evaluation are separate digest-bound stages: + +1. `experiment power-plan` accepts calibration observations only. Every + economic pair must contain exactly one `FrontierDirect` and one `NeedleMiss` + record with matching corpus, campaign commitment, task, route, split, + repetition, and pair seed. Failed, low-quality, missing, duplicate, + non-positive, mixed-material, non-beneficial, or zero-variance calibration + fails without emitting a plan. +2. A canonical `needle.power-plan/2` records the calibration-input digest, + estimator revision, alpha, target power, per-route log-ratio diagnostics, + required holdout pairs, material classification, and its canonical artifact + digest. The final gate verifies its campaign commitment against the exact + campaign bytes frozen by the manifest. The frozen schedule references the + exact serialized plan and must contain exactly the required holdout count + for each route. +3. `experiment final-report` accepts holdout observations only. Their identity + includes corpus, schedule, power plan, task, route, split, repetition, pair + seed, and arm. Every scheduled identity must appear exactly once. Explicit + failures remain in the report; the evaluator never drops them to calculate + a more favorable ratio or interval. + +Calibration values cannot contribute to the final ratio, BCa interval, quality +gate, or staleness gate. Holdout values cannot recompute or resize the frozen +plan. Synthetic plans remain structurally testable but always fail the final +economic claim. + A future general claim requires: - calibration and holdout tasks frozen before final runs;