From dac9a9ee1d2a12021b514437eda89523dda92ac7 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:23:34 -0500 Subject: [PATCH 01/10] SLM-301: prompt-contract observability with matched slot-inventory arms - coverage_class per frozen row (OBSERVABLE_PROMPT / OBSERVABLE_REQUEST_ONLY / UNKNOWN); all 51 rows are OBSERVABLE_REQUEST_ONLY pre-inventory. - Matched requests differ only by the production ensure_prompt_inventory suffix; request-derived, no hidden gold channel. - X22 deterministic arm: paired v2-strict delta +0.3125 (5/16 vs 0/16, Wilson [0.142, 0.556]) exceeds the predeclared 0.10 minimum; no regressions on observable rows. - AR tiny baseline: 6/32 decodes completed (all v2-fail partials), 26 decode_timeout (600-step checkpoint constrained-forest cost per SLM-294 evidence); reported unmeasured, not evidence. Rico rows not_run with SLM-294 cost evidence. - 100-record blind slot-observability audit: raw agreement 97%, kappa 0.0 (skew caveat); finding: v2 coverage detector misses inline slot enumeration -> false prompt_contract_unknown (append-only; no metric change). - 8 harness tests; new component harness.experiments.slm301_prompt_observability v1. --- src/slm_training/resources/versions.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 31b0b7bbf..40e8c67b4 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -4502,6 +4502,25 @@ } ] }, + "harness.experiments.slm301_prompt_observability": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm301_prompt_observability.py", + "scripts/run_slm301_prompt_observability.py", + "tests/test_harnesses/experiments/test_slm301_prompt_observability.py", + "docs/design/iter-slm301-prompt-observability-20260724.json", + "docs/design/iter-slm301-prompt-observability-20260724.md", + "docs/design/slm301-slot-observability-audit-20260724.json" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-301 LAR1-04 prompt-observability arms (as-is vs +inventory) with phase-0 coverage classification, AR/X22 baselines, paired predeclared-delta report" + } + ] + }, "harness.model_build.eval": { "version": "v52", "kind": "harness", From 8c1c2d9482689d06d249dff4f132a75247a71dc1 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:25:02 -0500 Subject: [PATCH 02/10] SLM-303: decode-budget/harness-interference audit on retained checkpoints - decode_outcome.py taxonomy with strict precedence (fallback never counts as model success); per-request budget/elapsed/forwards/ verifier/fallback/stop-reason records. - eval_runner additive integration: details[] gain decode_outcome, stop_reason, fallback_used; suite metrics gain decode_outcome_counts. - Census: only slm230 committed-SHA verifiable; hash-pinned the two remediated nonzero-timeout checkpoints; scoreboard classes separated (runtime_timeout/fallback/unmeasured/model_behavior). - Preregistered 1x vs 10x budget sweep: all 12 cells not_rerunnable (v0 checkpoints fail current output-contract check); the 10x-flip question is honestly UNANSWERED by re-decode. Recommendation: retrain the remediated recipe on contract v2, then re-eval at both budgets. - 25 new tests; harness.model_build.eval v50; component harness.experiments.slm303_decode_budget_audit v1; gates.ship v3 threshold mirror fixed. --- ...r-slm303-decode-budget-audit-20260724.json | 24024 ++++++++++++++++ ...ter-slm303-decode-budget-audit-20260724.md | 54 + scripts/run_slm303_decode_budget_audit.py | 917 + .../harnesses/model_build/decode_outcome.py | 140 + .../harnesses/model_build/eval_runner.py | 93 +- .../harnesses/model_build/ship_gates.py | 2 +- src/slm_training/resources/versions.json | 25 +- .../model_build/test_decode_outcome.py | 115 + .../test_run_slm303_decode_budget_audit.py | 159 + 9 files changed, 25524 insertions(+), 5 deletions(-) create mode 100644 docs/design/iter-slm303-decode-budget-audit-20260724.json create mode 100644 docs/design/iter-slm303-decode-budget-audit-20260724.md create mode 100644 scripts/run_slm303_decode_budget_audit.py create mode 100644 src/slm_training/harnesses/model_build/decode_outcome.py create mode 100644 tests/test_harnesses/model_build/test_decode_outcome.py create mode 100644 tests/test_scripts/test_run_slm303_decode_budget_audit.py diff --git a/docs/design/iter-slm303-decode-budget-audit-20260724.json b/docs/design/iter-slm303-decode-budget-audit-20260724.json new file mode 100644 index 000000000..42b2624fa --- /dev/null +++ b/docs/design/iter-slm303-decode-budget-audit-20260724.json @@ -0,0 +1,24024 @@ +{ + "schema": "slm303_decode_budget_audit/v1", + "generated_at": "2026-07-24T19:08:59Z", + "predeclaration": { + "experiment_id": "SLM-303", + "hypothesis": "A material share of historical zero/interference-flagged scoreboards are runtime/harness budget artifacts (decode timeout or fallback), not model behavior; widening the decode budget 10x flips timeout-classified rows to model outcomes.", + "falsifier": "10x budget does not change any timeout-classified record's outcome on the hash-pinned checkpoints, or the census shows the flagged scoreboards were already clean model behavior.", + "arms": { + "baseline": "recorded canvas cap (256/128) + inferred 10s timeout", + "budget10x": "10x timeout and 10x canvas cap; nothing else varies" + }, + "budget_evidence": "inferred from recorded eval_smoke.json: all 3 smoke rows share latency_ms=3333.79 (one chunk of 3 killed at ~10001ms \u2192 10s timeout); the original CLI flag was not persisted (config=null), disclosed as inferred", + "records": { + "ids": [ + "smoke_hero_01", + "smoke_button_01", + "smoke_callout_01" + ], + "source": "src/slm_training/resources/test_seeds.jsonl", + "disclosure": "the original eval suite directory is empty; the sweep rebuilds the same 3 smoke records (ids match the recorded scoreboard details) from test_seeds.jsonl" + }, + "seed": 0, + "fixed": "same checkpoint bytes (hash-pinned), same records, same seed, checkpoint-declared decode policy; arms differ only in budget", + "checkpoints": { + "ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "recorded_canvas_cap": 256, + "recorded_decode_timeout_count": 3, + "selection": "only checkpoints with ACTUAL nonzero-timeout scoreboards" + }, + "lexer_ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "recorded_canvas_cap": 128, + "recorded_decode_timeout_count": 3, + "selection": "only checkpoints with ACTUAL nonzero-timeout scoreboards" + } + }, + "excluded": { + "slm230_bounded_recursive_r4_r2": "zero-timeout scoreboard; budget interference not implicated \u2014 not needed", + "e173/slm294 and all other local checkpoints": "no committed SHA and no hash-pinned timeout scoreboard \u2192 not hash-verifiable for this audit (slm294 additionally feasibility-excluded: >280s/16 tokens constrained decode on rico)" + }, + "claim_class": "diagnostic" + }, + "census": { + "schema": "slm303_decode_census/v1", + "generated_at": "2026-07-24T19:08:59Z", + "taxonomy": [ + "model_valid", + "model_invalid", + "model_abstain", + "runtime_timeout", + "fallback_output", + "harness_error" + ], + "checkpoints": [ + { + "path": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e173-schema-context-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "5a84337262a851cfb74c7fed0e8c5d4f49112c75f7a82f4d3c0030d6f5c27f7f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "15a95b415884f59ffafea3e69826a41534b15afb72aeccecb85cd2099108f3b9", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "ed96323a4673c1ef4b6aa9c2c281e0b28a67c05ca0c7ee245ef8b06524ec87bc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e175-retrieval-8step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "ef8497ea80cd413e0c0e47f2d317ec6a331a8353ea94a11a16fd2feb682f0a34", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e175-retrieval-8step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "4e5b59d4abee56d3c155d9fba9c9d1503207f7dc275bea7aabde32b27023f3df", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e176-broad-corpus-8step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "996a5a43220b8c0216d389164b9056d8817e1fa9523ed88ff0560f6cee6c4ed1", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e176-broad-corpus-8step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f053e1604038c30afde727a3e2f73637ddb28624eeab7268e2f8ed0ac6188aa5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e177-semantic-judge-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "afd17c1ec60b0cc8540c09a4c66896c1b8a69ff44acf99c7ad9e754f1cbae81b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1ef4a2bfe6fb545e0bbb32ad811b515538dbc60ff13b64d3576875545bdedcf8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "0b6bb40e9968eb659ce668c34ecd7ece414eb3cf6d59e7769807e4d3e51269f3", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "453208f977d95b1c97d1c776558ddfa8fae000151fbf99a4ef3440f443deeda0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "22b8bd63e06670065e06b70f574c0a0123a831ec7324346c6041138f3010dcac", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e195-stratified-compiler-aligned-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "f8eb684b48dc34b90223fb8ecadf1210e5b0da702d3611b18e0f18d63e5b127e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e195-stratified-compiler-aligned-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "5db0f544a1bae7f9bac0386fd997e0ab53f877b8efc25df65083f6856952b5a7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c4308ed8ad6765a6fe44c0c21ffd9c5feb5b4c03ba28e0ebcb21a5fc0d86f01d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "53f14d5b8470cbe25e7fb1f97982e1978ecd57035149eb8a747f5a79d8106f30", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9259d61f2d30c1b08ad37f1266778a203efdfbe154de602a58616f68d069565e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "3bf8b8d25309818e438fd2814796edf3bd5825b454fd900ffdb80f2989615e4f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e8e0ee6ef5b79d930ac6caca8693dd1aa5ded6039b2fed44511c71d0f670d005", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "6674261fe499272fd9c108fdb9985ae33029248ac4611f80b4119941e472c0ea", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "21730db7515c4a1c82a060385e8b90520b7c8bbcb14925a310fce679a052ac48", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "3e1463436a28fa1d12ef28451c5aa90d186d0254619e8d50080b19a4c9ea2c58", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "012cfdc64922172d68219ce2b8691f43b4f3ba44069a86e34de1b78fb8ae859f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "8f700fa730cda5f67b6184be3a54fc36a754ff2a84fee8038b3e85b3dca3eb37", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f5783241ee15a8df6aa520e97c628d9c4d17f579a1dc6f7217ed1efa65483b7b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e219-schema-normalized-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "48ba5a7aaec6507a75b403d81603f3249ae5cb94b578217ee22071ed8d49e112", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e219-schema-normalized-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9f39de33fb935b10028d4973f8ec0e5fd1bced8b2d07ba150d840c99a520929a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r2/checkpoints/best_ship_score.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "0ebd083535dbf7391186d8f490c54da36862a29dcc5dbc92920e9b237e3c3c7a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r2/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "fe91fd46b1b0df6acb614657706d4af256b2fbf77ef0787d945c73e42e547894", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "8b46369bc11d0828ff1c7d400db05ede3e09d927a506a2f21a7ffbc31975d936", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r3/checkpoints/best_ship_score.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "55c5064dbad0a3803f7c742e2261e03f1a155c1950e5c3f2791b4072a5ab0a09", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r3/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "4d003ae1dbf4eb2dabcb3abbd0d60a09148bbbb55bbcbe83bd08f37f0569172a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "eb2b557882bc4de7c1b9b4cdde5595a1c429c355cde52604bf6903c3489f1cc6", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/best_ship_score.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "40a157e225295263c31ec7834d78a944faa1af8d5fa0a175288852ebedf96042", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c138e55d30cb4af12721374a337babc7cb8d9cde966cd0299f0d2b0272c1e136", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9e2bebb8c625af30493e51b9c6f6aba576624d700cf8417741f8c10353fc7721", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/best_ship_score.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "89798872ec110de08bb4fcf80d23fb08e9958d5e93f47199c825444b6c3b4ed5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9caa0413522d3e5d8a3cc56f44f1ef944e229c4d5cf75aba10224c0088ef92a9", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a7851d8d4efd113153733299203922ac7164e0b2ff8b7ed33a7754aa822b168b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/best_ship_score.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "5047ca1870e801aa3345c277df5d687aeffe9694f5f2ccf6eb4d3c5f3b7137cc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "903974b672a0a83d6f1b5a88a6a6be0c0807dfd6c6273bdddfa46b5b6c2e6c4f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "2b19cbb7aec97efb61100d28b00cefb616249932a917b73cb235fd5d912bae7e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6187e8c00baad70d9491f2b9a5ffa3cbe887f04d84a86df571d094ff225cdc0d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-fastpathaux-telemetry-probe-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f7628f8166f89250140b51f2e1b01b75f7a9a19bed0bf1ca2371d5121bf60947", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-fastpathaux-telemetry-probe-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9fbf12ab764fc286864102a1d543e2bc929b5394e565f3d3e7b9f8790d653343", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-fastpathaux-telemetry-probe-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "fe09a3b68b1dc7aea76c0707ab8ed550b25bb62c42be5f4234c2dd0aabfc9f2d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-accum2-4step-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "931d8d004a0cf6d42da92fbda9978036abda2912db1e0f95730562cb755c9492", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-accum2-4step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e75b81f503ee04bae738a92afc25ce67af5396f0ed7dd70f58700fa548fd1885", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-accum2-4step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c6e4ba62b598325df995c71bbb2bd8257d6536b186500b12ba4d9858de4065f0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "27d9be59abc3420875741f34795b2da4b8122a927fa47e9e583ab28fda241a88", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f98222f96bdf0203cbee36fb5a20458b2f7924b049ec1946120f9d13a7343aa5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "cdb767271062604668cc1ac5d6981fa55da70a3c75c94ec85087cf2602ac33f8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f943321f47b5c50c687921fcfd6516aed77a97d6201903ae0a3756eba93cc0a0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1836a63c4caf2215f5a5ef8cf2b9a225454e074a37370fde80693c58bc87c745", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-seed1-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "b160ec7745b5d01fb367a5acd646a97fabd4f8cb080d4198935a175d7c9e7ea7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-seed1-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "43d21f2527f2d0270caef0ee7d6dee9861a0d367852e819281ede9f81d11aaee", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-seed1-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e33b5f9596830f50cb8c724733c4ec5684d3ad96495af89000e7675df1723411", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-no-midloss-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f98222f96bdf0203cbee36fb5a20458b2f7924b049ec1946120f9d13a7343aa5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-no-midloss-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "37c98f3a5be830281f23a6a60ee7a01d4f51111389e51d02fcb7f17024d9837f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "18646410c139267f2f558147032ce5327543fd784b418e30ee9bda7e7d6c2602", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "8b48b37cdcf5528e75c77714ada34b04a48338bebd8c8bbf8d2074d07095eca0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "b2efa28eff0e54449692cc6c6ac125741b6436e892c8843f50ca060226e6e7e7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "80be2bb2b70027162f26f5a90a1816a0bf43002bfd33224f9ac147aa5e4a97ae", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1329222d34c3097e4bb2b26a5a505c7821d31d4cc4582b69dc90eeae7b4ece7f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "bd295cb96ce9dd5551d15c4205c5f96f8117b969ca518a1865fefba4e3d847fe", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f7ac95fe6f46c59f85234a8df0cc79989988c14437248bd28396055d31652e5a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c08fd319cd7b72e19990a63496dcccb5850282c11cfd9095817bd92a87848afb", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "90d365b6788679cd33908be6316f9dfca523c1e02beb15319cd86fb74b8ffe92", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1a9d63aca9ce164eb13b8f9609f8510f9f27b8aedf9786947a8e5c0ac30e3385", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6bcdb828756916d436daae07c96874d12ffcf08ff211fcec923763c433ff2ed5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3fcaff91b4bfdbb0b847958f87699f0432faf5f14ee94c49d0d29cd34cce7c6e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f788d6d0c0936d6d2c77314a880e90603889534e6bcc96b466be7bc29b44e637", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "da1cd4d759fb11c056d69f701cdf8903d2dd1f9d33c98847b25c758b4569c9d8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e0ad786c8eb50c2f7436ff8a4c2e1dd04043d06916ecfff489d93b99785982f1", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "083e0d49d5630db7fac4e0f09189449dd5fa40767db9b273c2a381d2742916b2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "33a8511d8dfaeb50072f42e29b8c5fb23d30b47012e98586491291125fafdfcd", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "52a5c61e7d739c7f670fbb3eecded0066e933d7ffc502420a2b490df958b7b17", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "738ea77137d40b023a74d450d6eaf036c0be8cbf0699f0ed516facd253c68d5b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f749c34519fdaed7b096ad1c2f977217553e5ebfb6209bc4ded262e5b4fac2e2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a22596f72aab3a1bf98a5022a8a9c132a66b46b16df7ac1609792ed9713eed60", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6c1da50b56f463e9e4782923cdf4a74e83737f8ebd303c1e775c2775bf56623d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "0cbca0cf73c33d7609881db5f832f4884faf5b6a67dcb0ac778c8edc3d98e82b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7be4ccaa98f4aa071843219ebe11ad0823c6323a1a8a49064aa43869eca9cf10", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "b57e68b1a720e35cf16f98a6f8519470e36c39b6f41d781c6bb7c9a908709bff", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f01a03aaeff363b9c4cdf8ea858653b1dd8dff0485480384f6cfb636dcef5a4f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7ef101ae1c1859c9eb6561f47e37b6dada9636a346667f1270800222b5362147", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1ea1669280c3c2fea4073e08f542867fb4aa51f85066f170218628fe69d50e7d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "306449af4a7ed734fca8f2b9f29e698759a31c29c586e139d960592c67203867", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1e27051441954b7f1f5c2fbcdf38b7d02ebac53c97452a24239b0a4b8a45187b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "03312d59833d17dfd2c07a6f3be09e6e0aa8303e5dcb1377c28921b9b4b87afc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "df2181c9a57eb341af23ff22795f32275f2c7cee6c652b6506ad2c37f63588ae", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c6c96046ab21c921edaded81a295b7126d821da3c466d69839bbb6aa62b6c627", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e8bba479cd2a002594bb4c3ffad836fcfdf7ef1c606b858c97d652988866d791", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f7982b7635b496c63ba8c09e0f6ea6127b93ede103acd268874a8848251cdfa8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "14577f9b04578758919692921c7c1048c2c34fe43b8a06fdb68e3304351a1eae", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f7982b7635b496c63ba8c09e0f6ea6127b93ede103acd268874a8848251cdfa8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "154f205d4e9232bcd366db15b46c38abd2ef21fd0a3f595b8ca6aae6c20c0e72", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "10b05f6f153a101bac1b22979e010cee506f3c313fd34f42f6ba245d7783ff7e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f814e59ba497f68284928dd9b93210e49e3ca0e8bab676fa5971e95555c01ef4", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "dacaa7df813ee99f537d4836751a8ac45d1015a0f69f427ae04f1e3e65d56841", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "bda7e7f35134f3f9a6ca1d84b7d970250f2246951c71ae7bf4b563d023fbb7bf", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c832ad3dabb4fc2416a5ec8f7fa1f49922e1b06128f12f63637d486a6709c656", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "62e00d9477b5fa44b3691fa6c1d697b518c29fc2eea19f961114d7b538027b17", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6d485bae0c32092454794f41af96abf875465f1fd82f9ff12c67c868d596609a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a22bb88120ca3dea6f51cf1d599a667817bdaea823cc199571c0b79cae65d1b4", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c46625266b218219337bcc7511acb0a8cd59dd916aa720a7510653baeccfc27d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "2755ab5f7b700dcc5cda01980df931dcae311cc41abc181a9ba49e346e99986e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "0037e12690677c3848e629816ae3d0da59fa19557e394c6110f992464500a3cc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a2c4d87e89f82b0a76133a636be08adfdc4698b4cc2482222cdb9bd97ec5b398", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "91f22e60151c7506dd7d64c3d427f4bc8466b8f8361e29ede9d884a83c79aaf7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "271d1f03bbd31a29bfc760f6c77901fb8d9efea5d54600ee841a19e9c7824df2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "2eb3efe8b9ad0a29c2b401429785035b185c7323bc6b86efc7e5341d5a1fbe50", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "df159e8ed49ad1107fca9017ce369435e991930a625ef96193289084df34f3dc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "315235ae4ef1bca1912eb5f1227004fe44061564f87e4063932f7b2fbf52027d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1ff4f214e20a122c923561d1aefc15550dfe90ee108808ab691a490f58bc4dd7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c5ae47eed8b36ba070b76734cb503385a5a61791ebcbf2e2635bf86148c56a39", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6f1e3591829764c81cf412bd8bd0680614a517297c69c0d4fcb81aed3f0e09cf", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7aab682d7d30f9341ffd756a0cf65fba62c887ab3455f36b3118808e10d454ee", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "600d69f324fa4b6bd16d9d9ecd5d5c8f604f7fba5f26b431a22b2d6fa8918949", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-interval32-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-interval32-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "d8bb2d26e0a21108569ece5177cdef0de0feabf32bb101bad4b048619273fcc3", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "777924cad2660579359fd8b1dba1961c83ac54f0c8fb7bb8e12e483eceeb1f2c", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "2f9bcaa9c2176de6438807b9493fb3ced19a5cecd8ffa084761ce810e9a01610", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7d85b6c7265f53cca4a5353a98a908ce692ff80a7acc1c02f29e91f889452a4b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e7cc5615002773ac11c15eeeda251dd7c016e0eb03b12d9f4775b5f104c8fc86", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e31fcfd2b03e6740ead7ce691ebf373e16de908b366b190117f25c04cf38161c", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "22c1a664f3e0540d20302b80f9bb9b7509b5ac13378c6ba10cbf9bb54cdcae58", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "25046f497d7edabe96e61af1b1b8d214387f65fe3de08a6ead297ff85987eab0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "76f4e74dbbf9ccb5d5ca0a2559384479f577c3ca5c581c41692ea9bdc480e862", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "95d8868d59df45dd7c22a27d09b7205ce375c9ef97aaa1d8070e9728cfea7237", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "d1e2ad76f953d10aba20db456b7de18ae918d6ae481aed0aee438ff9361187b1", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "bd295cb96ce9dd5551d15c4205c5f96f8117b969ca518a1865fefba4e3d847fe", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "71e95b35523236c762e25c20e226bba71f43d6916f759df6ea3fff34942e838e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7db15eff2a3a73e6372d912222667208b2bd8fc8493beb7638565e6385a792d4", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "aab63835bc695d5d0a098c2659bd793f64fd47d722480677d0ca5a551a60b805", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "23d7a9797e351cbfc8ca92bf2abbc9b2684be48648fb9836649dd2187a13c841", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "fd15d55eb8451ce089ca0299f8cbe5962a07f6fb71790dfd51c5b14c30309f4b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "b8f4abb0f5f8dd9b54b5df36cf3f4c9381c541b60250fdfe8b60af81c5c79c37", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "d210c025979c64d86b4f3dd553b3d8c0417a176ba4fb81e4ea7541106bf4d7e6", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "bd295cb96ce9dd5551d15c4205c5f96f8117b969ca518a1865fefba4e3d847fe", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "88d9e56ee589035d3adbc928cd7f7b9e62d7911b0100f0609b2153210e0da04a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3b5281385368b5a575b0766c2c8a8407a45527535cb055751c8fd74222d624d7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3be97e80780de07a6e2ffa4f73f915fa073c40f0aecba32ab4a8f9cde33200e2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1735aaa3bf0c28c41f133648c4c83f37e372b6015263687b5574553cb4a4d398", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "80be2bb2b70027162f26f5a90a1816a0bf43002bfd33224f9ac147aa5e4a97ae", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "162f43cecf3f7e0053a7ff8d3dcf4a7fd2a520c55daba01d0bbe66ee6464f145", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "fa24f6b2f5c235514f0f01e6fff567d24054a59974e81593423d4ad2e636f075", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "db9f2227d01d01a9aefdb5274858fbe1b326a1624fc8ebba25bb9acb43a47ffc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6b7b68b6cba2f4e4771b0103a595fb744cdfd1634e70dfa66f65f32a017f318b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f423835433c85560049881bfa8186a1c46715ecc8f6e7cad8aa24b9c65915674", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9a2e3f51500646551799945dc39cc4584658e8de01d70110845c31bada55b772", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "12c65f73f254879511a67b992c47b338571d6b86dcc4336237eb0d4abe7c35bb", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f749c34519fdaed7b096ad1c2f977217553e5ebfb6209bc4ded262e5b4fac2e2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7c835fe68c73261142df61b27541988276a103786adbe92eb5253ec689372be1", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a21e92839eea9456397c51644a22e5fcf2314356f103f584b9f5194a6f175fe8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "da923a2d8d53a60e00161d9ff5dd3cac08825f16a99349890f68858ef2042ed2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "2274d2e172c1e90225f170ecde3c3b039f4d5965d34b13ba4ef3164caf51d778", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "21f3b7b6a54efc587eedc1e586a4331e60026c27fcb7e7e0d07a479348b27743", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9992d9539e3cd68d904889238da4f0072cdb0d2baa7abff40387e533618bc05c", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "00c6a23f363be27a1013b279411703b3872e4444f3c59baec15a67d2dd968ba8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3fb1817829503f2fe7b028c295729a20bddf252864d95981022f2c5f56bf14bd", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-4step-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7874b2817373478543de0bd82dddf3688d7e0449eb1940637f49904b98e9fea6", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "00f4437b53fd11034ce59601c4a4ad2677ef2749bd9d8619595b9f1f696a09e4", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-4step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "27359580f640e7b30b8c166c6676f7a9f44c231958a8f2fd4bf8ebd22b4627e3", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "dd62dcae43bfd34874393125631b82e2fa7f40073718f05a3f360e9f01825939", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "47c0a8a01679837024abec0fa38f69ff9a7418687cfffa73383ed2486ab11797", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1ead3aef720d766851672482c84395e8a1cad2da341c22359e32cb7920b91617", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a980c62056893817f8dc70c2d50010192a0f3be2740d43a43a391b4782a2a014", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed3-batch8-4step-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "55579879eb4b366e3a9b2b695ef412b4cb519c5c07a7c137eca1c0bb150ca6c9", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed3-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "ca8662d472eaa470d3ad7c9fb0004bacbb488b62d01a957272f3d35635e4bbac", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed3-batch8-4step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "393cdeb5149ca42431840041f5add286a0f3a8649e2de330f98479935d75447b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "ed9f257bc9463368c07ea54179dbddae69ca41ab2085f7cb8aed75d7b6aadc9e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f5bd15e7b5d508b398962363aff5364bef38372b60ed2c362b1dd9e4a96031d6", + "committed_sha_match": null + }, + { + "path": "outputs/runs/local_directml_adreno_20260714/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "932654f983678e376d9d4a46778b6311b69fbab498d4812fda5c046ac65b7d9d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/local_directml_adreno_20260714/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "162e3916c2cf4bf269fdd1c53e0bd69c4931cefccb71dd265aaf9d198f098b36", + "committed_sha_match": null + }, + { + "path": "outputs/runs/qx_e42_factorized/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "390c2e4628064480f3607b07c12c9010016a693a20113a32d95a3c09569fbee6", + "committed_sha_match": null + }, + { + "path": "outputs/runs/qx_e42_factorized/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "dffb7f1de73553bf2385e5cd74e67576b33478d5880e097524b5698332c49e80", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm230_bounded_recursive_r4_r2/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": "1604b2cb\u20268b28a", + "actual_sha256": "1604b2cb9282928fa0969ecbbe7d78c9aa4b9907f74d0d58936bfc298a88b28a", + "committed_sha_match": true + }, + { + "path": "outputs/runs/slm230_bounded_recursive_r4_r2/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6fb54ccc01d016e9c38445ce3aeb7cacb30adfe06efb3e437ddd48a55275fdf0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm262_cpu_smoke_cpu_smoke/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "ec1fff0540e4cc07227013b7111d6f602b610ffc977ffaa8a4656f45606948e5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm262_cpu_smoke_cpu_smoke/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9cccf95a220eabf78aa1042d24adfbe78dd029e5a5490555a439ca880cbbb72f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm262_probe/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "8727bbdc6f7b1c2d226c91f0b585f50d49e8f9bf4340f72ccc87dde8bdd29029", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm262_probe/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "90293f519bbf871ead925c6a3799dc4c18891a9abb9214a9ca73a4f43c9d46d7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm294_tiny_baseline/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "f8017ac686f1685bd3b741b8e88f650409f9c34dd184844bfa40c198fa60beab", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm294_tiny_baseline/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "509197b823a80a6ca821022bc44d8f800d4633ec5ac31cdc126ad182782b78b1", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e499-choice-compatible-strict-hf-choice-candidate-r6/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "7230ace9\u20262e2fab", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e499-remediated-roots-hf-choice-control-r4/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "bb4bec5f\u2026f359fb6", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e499-strict-r4-hf-choice-candidate-r4/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "81b2cb66\u2026bcfbaf1", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e500-document-control-hf-choice-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "a40f39a5\u2026772d6834", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e500-document-control-hf-choice-r3-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "9f752ae0\u20260b2b53", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e500-documentized-expression-hf-choice-r2/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "f54cea08\u2026773d3f0", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e500-documentized-expression-hf-choice-r4-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "a0ed6a58\u2026dda5623", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e501-e396-e500-init-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "f86b83d3\u2026cc9cf15", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e501-e396-e500-uniform-init-r2/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "14605459\u2026736e4e7", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e501-e396-e500-uniform-init-r3-1k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "d84d34c0\u2026b5be2ffd", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e502-e396-e500-prior-retained-lr3e4-r3/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "e1e833cb\u20260746cb6a", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e502-e396-e500-prior-retained-lr3e4-r4-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "6f937374\u20264a46a726", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e502-e396-e500-uniform-lr1e4-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "fcd51266\u2026f047255e", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e502-e396-e500-uniform-lr3e5-r2/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "528c86a6\u2026a62677c", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e503-e396-e500-retention0-r1-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "af6e9b1c\u20268a0af431", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e503-e396-e500-retention001-r2-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "7c5f016f\u20261be75711", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e503-e396-e500-retention003-r4-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "2dbb52db\u20265751455b", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e503-e396-e500-retention005-r3-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "4093f1aa\u2026af8d2031", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e504-e396-e500-replay000-r1-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "35cd38e0\u202656334c87", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e504-e396-e500-replay0125-r2-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "da63b403\u20261b725d3", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e504-e396-e500-replay025-r3-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "91ab3f73\u2026c3d85b4", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e504-e396-e500-replay050-r4-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "7d7e056e\u2026c90294f9", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e504-e396-e500-replay050-retention001-r5-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "1fc2fc23\u2026a36036c", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e505-e396-e500-replay050-loss-attribution-r1-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "8fd11acd\u2026525967e8", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e540_training_loop_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "46c1d82d\u2026589e6014", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e541_training_loop_twotower_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "ffc8f2e3\u2026db3ffbaa", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e542-e531-root-reference-arity1-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "2d5cd4b3\u20266854c5d8", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e542_training_loop_twotower_resume_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "682ab617\u202677347c42", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e543-e531-root-reference-bounded-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "c6be3791\u202651d7f90", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e543_training_loop_twotower_resume_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "6219feed\u2026130fd1da", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e544-e543-root-identity1-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "3b6e3c00\u2026474f20c", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e544_training_loop_twotower_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "8531f7d7\u2026c2eb8036", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e545-e544-root-identity-neg1-control-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "9e54d470\u202676fa1", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e545-e544-root-identity-neg4-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "14dd4404\u202661ae", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e545_training_loop_twotower_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "3d08f592\u20261823e1fe", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e546-e544-strict-subset1-control-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "46aba904\u20260fc55", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e546-e544-strict-subset5-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "a1a6bfc9\u2026b4efe2", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e546_training_loop_twotower_scratch_20260720/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "dfc0272c\u2026351cf9c5", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e547-e544-strict-subset2-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "37002bfd\u20260fc57", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e547_training_loop_twotower_scratch_20260720/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "c67db09f\u20266d17627f", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e548_training_loop_twotower_scratch_20260720/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "6581e32f\u202681386936", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e551-e544-strict-subset2-no-lexeme-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "e7921e66\u2026dac32fc6", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e552-e544-strict-subset2-lexeme05-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "49a9c111\u2026a151fc04", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e553-e544-prior-proportional-r3-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "510e55cf\u2026e75399d", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e554-e544-slot-next-context-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "af3cbce7\u2026c67b579", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e555-e544-slot-pair-interaction-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "af53e161\u2026addf19e", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e556-e544-slot-context-combined-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "139c670c\u20265831f0a", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e557-e544-slot-pair-balance1-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "438d9871\u2026b97db05", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e558-e544-owner-coverage-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "8a572738\u2026de85382", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e558-e544-owner-coverage-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "a45909df\u2026381ede", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e559-e544-owner-coverage2-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "1d11926d\u20269aac861", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e560-e544-owner-threshold4-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "dae11cee\u2026d7686a3", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e561-e544-owner-threshold7-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "35a4fe6d\u20263a127f9", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e568-e561-cont48-r1-48s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "8dcc0804\u20260283a12b", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e569-e561-matched-cont48-r1-48s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "8254fcf7\u2026c6535f73", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e572-e569-fidelity2-r1-48s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "bb6a58ff\u2026cc29efa2", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e573-e569-fidelity1-r1-48s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "ff21fc0c\u2026cf59070d", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e574-e569-slotloss2-r1-48s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "649cf512\u20263810b7c2", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e616-object-property-slot-bias-scratch80-20260720/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "119dd41a\u2026eef0c508", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e620-required-slot-coverage-scratch800-20260720/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "3ce5c9ef\u2026363ecc5f", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e714-symbol-only-scratch600-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "71ef1d25\u20262b49e", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e720-symbol-only-component-inventory600-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "842a1a21\u2026f91a11", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e721-symbol-only-component-plan190-r4/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "c30fd565\u202620f530", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e722-symbol-only-component-edge150-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "08873bf0\u2026a25597d", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e723-symbol-only-slot-owner140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "787d2d21\u2026bd62795", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e725-symbol-only-component-inventory130-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "897208bf\u20268ba1e", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e726-symbol-only-root-arity140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "d84148fe\u20261b91b", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e727-symbol-only-binder-arity140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "c211d2ea\u202636a857", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e729-symbol-only-binder-topology140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "c5bafb8d\u202618c43d", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e731-symbol-only-root-arity140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "bff1e0e6\u20262fbb88", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e733-symbol-only-root-identity140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "cddb5f28\u20265167fc", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e735-symbol-only-root-arity-fullhead140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "710e2dbe\u202668b970", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/gx_x22_kapur_tree_edit_s0/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "a9cfb450e8146089cb26b6df84e90a5073627c4e59a2933d16f69034ec802ff6", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e120-unsandboxed-20260715/e120_unsandboxed/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e121d-e53-judged-20260715/qx_e53_honest_v5_champion/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e123b-judged-20260715/e123_judged_32step_b/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e127-schema-slots-20260715/e127_judged_schema_slots/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e128-schema-slots-20260715/e128_judged_schema_slots_64/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e129-schema-slots-20260715/e129_judged_schema_slots_64_lowweights/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e130-schema-slots-20260715/e130_judged_schema_slots_seed1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e132-generation-focus-20260715/e132_generation_focus/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e133-no-fuse-ltr-20260715/e133_no_fuse_ltr/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e135-hf-context-20260715/e135_hf_context_control/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e136-hf-context-20260715/e136_hf_context_32/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e137-hf-context-20260715/e137_hf_context_16/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e138-hf-seed1-20260715/e138_hf_context_seed1_8/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e139-hf-seed2-20260715/e139_hf_context_seed2_8/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/overnight_retrain_1000/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/overnight_retrain_200/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/restructure_cpu_scratch_v0/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + } + ], + "scoreboards": [ + { + "scoreboard_path": "outputs/runs/e169-grammar-derived-20260716/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e169-grammar-derived-20260716/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e170-lark-state-20260716/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e170-lark-state-20260716/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e171-ast-boundary-20260716/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e171-ast-boundary-20260716/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e172-schema-types-20260716/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e172-schema-types-20260716/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-probe256/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-probe256/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-probe32/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 32, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-probe32/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 32, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-eval/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-eval/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-eval2/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-eval2/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-full/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-full/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-scratch-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-scratch-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e174-unfrozen-context-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last.pt", + "checkpoint_sha256": "15a95b415884f59ffafea3e69826a41534b15afb72aeccecb85cd2099108f3b9", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e174-unfrozen-context-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last.pt", + "checkpoint_sha256": "15a95b415884f59ffafea3e69826a41534b15afb72aeccecb85cd2099108f3b9", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e175-retrieval-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e175-retrieval-8step/checkpoints/last.pt", + "checkpoint_sha256": "ef8497ea80cd413e0c0e47f2d317ec6a331a8353ea94a11a16fd2feb682f0a34", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e175-retrieval-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e175-retrieval-8step/checkpoints/last.pt", + "checkpoint_sha256": "ef8497ea80cd413e0c0e47f2d317ec6a331a8353ea94a11a16fd2feb682f0a34", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e176-broad-corpus-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e176-broad-corpus-8step/checkpoints/last.pt", + "checkpoint_sha256": "996a5a43220b8c0216d389164b9056d8817e1fa9523ed88ff0560f6cee6c4ed1", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e176-broad-corpus-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e176-broad-corpus-8step/checkpoints/last.pt", + "checkpoint_sha256": "996a5a43220b8c0216d389164b9056d8817e1fa9523ed88ff0560f6cee6c4ed1", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e177-semantic-judge-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e177-semantic-judge-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e178-schema-arity-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e178-schema-arity-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e179-compiler-owned-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e179-compiler-owned-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e180-symbolic-root-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e180-symbolic-root-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e181-semantic-balanced-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e181-semantic-balanced-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e182-root-choice-telemetry-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e182-root-choice-telemetry-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e183-root-edge-vs-path-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e183-root-edge-vs-path-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e185-compiler-aligned-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e185-compiler-aligned-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e186-ast-complete-eos-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e186-ast-complete-eos-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e187-literal-state-sync-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e187-literal-state-sync-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e188-slot-contract-compiler-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e188-slot-contract-compiler-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e189-nesting-aware-schema-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e189-nesting-aware-schema-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e190-forward-binder-scope-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e190-forward-binder-scope-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e192-full-alignment-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "453208f977d95b1c97d1c776558ddfa8fae000151fbf99a4ef3440f443deeda0", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e192-full-alignment-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "453208f977d95b1c97d1c776558ddfa8fae000151fbf99a4ef3440f443deeda0", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e193-e184-generalized-compiler-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e193-e184-generalized-compiler-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e194-schema-symbol-scope-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e194-schema-symbol-scope-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e197-stratified-alignment-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e197-stratified-alignment-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e198-reference-complete-eos-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e198-reference-complete-eos-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e199-enum-progress-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e199-enum-progress-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e200-layout-declaration-role-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e200-layout-declaration-role-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e202-role-stratified-alignment-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e202-role-stratified-alignment-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e203-children-node-role-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e203-children-node-role-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e204-content-slot-contract-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e204-content-slot-contract-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e206-lark-terminal-alignment-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e206-lark-terminal-alignment-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e207-schema-enum-sequence-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e207-schema-enum-sequence-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e209-list-occupancy-alignment-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3bf8b8d25309818e438fd2814796edf3bd5825b454fd900ffdb80f2989615e4f", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e209-list-occupancy-alignment-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3bf8b8d25309818e438fd2814796edf3bd5825b454fd900ffdb80f2989615e4f", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e211-list-scope-occupancy-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "6674261fe499272fd9c108fdb9985ae33029248ac4611f80b4119941e472c0ea", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e211-list-scope-occupancy-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "6674261fe499272fd9c108fdb9985ae33029248ac4611f80b4119941e472c0ea", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e213-contextual-decision-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3e1463436a28fa1d12ef28451c5aa90d186d0254619e8d50080b19a4c9ea2c58", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e213-contextual-decision-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3e1463436a28fa1d12ef28451c5aa90d186d0254619e8d50080b19a4c9ea2c58", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e216-schema-role-judged-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last.pt", + "checkpoint_sha256": "8f700fa730cda5f67b6184be3a54fc36a754ff2a84fee8038b3e85b3dca3eb37", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e216-schema-role-judged-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last.pt", + "checkpoint_sha256": "8f700fa730cda5f67b6184be3a54fc36a754ff2a84fee8038b3e85b3dca3eb37", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e220-schema-normalized-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e219-schema-normalized-32step/checkpoints/last.pt", + "checkpoint_sha256": "48ba5a7aaec6507a75b403d81603f3249ae5cb94b578217ee22071ed8d49e112", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e220-schema-normalized-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e219-schema-normalized-32step/checkpoints/last.pt", + "checkpoint_sha256": "48ba5a7aaec6507a75b403d81603f3249ae5cb94b578217ee22071ed8d49e112", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.5, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_valid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_valid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval_held_out.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval_ood.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval_held_out.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval_ood.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.25, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_valid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.25, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_valid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-chosen-only-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-chosen-only-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-fixed-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-fixed-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-postremask-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-postremask-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-remask-fixed-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-remask-fixed-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-main-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-main-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-persist-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-persist-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout1-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout1-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-top1-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-top1-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-trust-fixed-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-trust-fixed-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-explicit-lark-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-explicit-lark-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-ltr2-step64-ltr-primary-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-ltr2-step64-ltr-primary-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-attempt1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-attempt1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-skipprobe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-skipprobe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-trust/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-trust/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-profile-fixed-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-profile-fixed-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-toygrammar-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-toygrammar-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-unconstrained-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-unconstrained-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-ltr64-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-ltr64-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-one-ltr-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-one-ltr-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-topk4-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-topk4-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c08fd319cd7b72e19990a63496dcccb5850282c11cfd9095817bd92a87848afb", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c08fd319cd7b72e19990a63496dcccb5850282c11cfd9095817bd92a87848afb", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "da1cd4d759fb11c056d69f701cdf8903d2dd1f9d33c98847b25c758b4569c9d8", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "da1cd4d759fb11c056d69f701cdf8903d2dd1f9d33c98847b25c758b4569c9d8", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6bcdb828756916d436daae07c96874d12ffcf08ff211fcec923763c433ff2ed5", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6bcdb828756916d436daae07c96874d12ffcf08ff211fcec923763c433ff2ed5", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr8-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "33a8511d8dfaeb50072f42e29b8c5fb23d30b47012e98586491291125fafdfcd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr8-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "33a8511d8dfaeb50072f42e29b8c5fb23d30b47012e98586491291125fafdfcd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-ltrdecode-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-ltrdecode-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7be4ccaa98f4aa071843219ebe11ad0823c6323a1a8a49064aa43869eca9cf10", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7be4ccaa98f4aa071843219ebe11ad0823c6323a1a8a49064aa43869eca9cf10", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7ef101ae1c1859c9eb6561f47e37b6dada9636a346667f1270800222b5362147", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7ef101ae1c1859c9eb6561f47e37b6dada9636a346667f1270800222b5362147", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-unique-64step-ltr2-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "1e27051441954b7f1f5c2fbcdf38b7d02ebac53c97452a24239b0a4b8a45187b", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-unique-64step-ltr2-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "1e27051441954b7f1f5c2fbcdf38b7d02ebac53c97452a24239b0a4b8a45187b", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-eosfix-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-eosfix-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "10b05f6f153a101bac1b22979e010cee506f3c313fd34f42f6ba245d7783ff7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "10b05f6f153a101bac1b22979e010cee506f3c313fd34f42f6ba245d7783ff7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-sample-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-sample-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-lexer-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "bda7e7f35134f3f9a6ca1d84b7d970250f2246951c71ae7bf4b563d023fbb7bf", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-lexer-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "bda7e7f35134f3f9a6ca1d84b7d970250f2246951c71ae7bf4b563d023fbb7bf", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6d485bae0c32092454794f41af96abf875465f1fd82f9ff12c67c868d596609a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6d485bae0c32092454794f41af96abf875465f1fd82f9ff12c67c868d596609a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-256step-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "2755ab5f7b700dcc5cda01980df931dcae311cc41abc181a9ba49e346e99986e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-256step-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "2755ab5f7b700dcc5cda01980df931dcae311cc41abc181a9ba49e346e99986e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-64step-grammar-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "91f22e60151c7506dd7d64c3d427f4bc8466b8f8361e29ede9d884a83c79aaf7", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-64step-grammar-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "91f22e60151c7506dd7d64c3d427f4bc8466b8f8361e29ede9d884a83c79aaf7", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-64step-ltr2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "df159e8ed49ad1107fca9017ce369435e991930a625ef96193289084df34f3dc", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "df159e8ed49ad1107fca9017ce369435e991930a625ef96193289084df34f3dc", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-simple-128step-diffusion-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c5ae47eed8b36ba070b76734cb503385a5a61791ebcbf2e2635bf86148c56a39", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-simple-128step-diffusion-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c5ae47eed8b36ba070b76734cb503385a5a61791ebcbf2e2635bf86148c56a39", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-best-checkpoint-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "777924cad2660579359fd8b1dba1961c83ac54f0c8fb7bb8e12e483eceeb1f2c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-best-checkpoint-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "777924cad2660579359fd8b1dba1961c83ac54f0c8fb7bb8e12e483eceeb1f2c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-constrained-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-constrained-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-ltr2-best-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-ltr2-best-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-256step-ltr2-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "e31fcfd2b03e6740ead7ce691ebf373e16de908b366b190117f25c04cf38161c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-256step-ltr2-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "e31fcfd2b03e6740ead7ce691ebf373e16de908b366b190117f25c04cf38161c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-32step-constrained-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-32step-constrained-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-32step-ltr-repair-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-32step-ltr-repair-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-constrained-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "95d8868d59df45dd7c22a27d09b7205ce375c9ef97aaa1d8070e9728cfea7237", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-constrained-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "95d8868d59df45dd7c22a27d09b7205ce375c9ef97aaa1d8070e9728cfea7237", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-diffusionmask-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7db15eff2a3a73e6372d912222667208b2bd8fc8493beb7638565e6385a792d4", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-diffusionmask-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7db15eff2a3a73e6372d912222667208b2bd8fc8493beb7638565e6385a792d4", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fastpathaux01-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fd15d55eb8451ce089ca0299f8cbe5962a07f6fb71790dfd51c5b14c30309f4b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fastpathaux01-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fd15d55eb8451ce089ca0299f8cbe5962a07f6fb71790dfd51c5b14c30309f4b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fidelity05-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fidelity05-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fidelity2-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3b5281385368b5a575b0766c2c8a8407a45527535cb055751c8fd74222d624d7", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fidelity2-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3b5281385368b5a575b0766c2c8a8407a45527535cb055751c8fd74222d624d7", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-lexer-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-lexer-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-retrieval1-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fa24f6b2f5c235514f0f01e6fff567d24054a59974e81593423d4ad2e636f075", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-retrieval1-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fa24f6b2f5c235514f0f01e6fff567d24054a59974e81593423d4ad2e636f075", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-schema-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f423835433c85560049881bfa8186a1c46715ecc8f6e7cad8aa24b9c65915674", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-schema-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f423835433c85560049881bfa8186a1c46715ecc8f6e7cad8aa24b9c65915674", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-unfused-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-unfused-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr4-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "a21e92839eea9456397c51644a22e5fcf2314356f103f584b9f5194a6f175fe8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr4-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "a21e92839eea9456397c51644a22e5fcf2314356f103f584b9f5194a6f175fe8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-batch8-8step-unconstrained-eval-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-batch8-8step-unconstrained-eval-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed1-8step-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "00c6a23f363be27a1013b279411703b3872e4444f3c59baec15a67d2dd968ba8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed1-8step-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "00c6a23f363be27a1013b279411703b3872e4444f3c59baec15a67d2dd968ba8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed2-8step-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed2-8step-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed2-constrained-probe-fix-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed2-constrained-probe-fix-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed3-constrained-postfix-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "ed9f257bc9463368c07ea54179dbddae69ca41ab2085f7cb8aed75d7b6aadc9e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed3-constrained-postfix-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "ed9f257bc9463368c07ea54179dbddae69ca41ab2085f7cb8aed75d7b6aadc9e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-stub-eval-control-20260715/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-stub-eval-control-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-timeout-control-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-timeout-control-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/slm294_tiny_baseline/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/slm294_tiny_baseline/checkpoints/last.pt", + "checkpoint_sha256": "1eb2b2df1fcc8de1ede419bf70ab64d3ff4e21d355e3d6d30895f4003ce109fc", + "suite": "smoke", + "classification": "unmeasured", + "n": 0, + "decode_timeout_count": 0, + "fallback_count": null, + "decode_canvas_cap": 256, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "outputs/runs/slm294_tiny_baseline/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/slm294_tiny_baseline/checkpoints/last.pt", + "checkpoint_sha256": "1eb2b2df1fcc8de1ede419bf70ab64d3ff4e21d355e3d6d30895f4003ce109fc", + "suite": "smoke", + "classification": "unmeasured", + "n": 0, + "decode_timeout_count": 0, + "fallback_count": null, + "decode_canvas_cap": 256, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e120-singleton-legality-20260715.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e479-e396-array-items-full-ship-gates-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e486-json-number-typed-any-bounded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 14, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 15, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 15, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e488-e396-json-number-typed-any-full-rico-merged-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e488-e396-json-number-typed-any-full-rico-merged-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 1500, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e489-e396-json-number-typed-any-bounded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e490-e396-json-number-typed-any-full-ship-gates-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e497-current-main-playground-provenance-smoke-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 1, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e611-repeated-plan-slot-allocation-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e612-authored-typed-array-nonempty-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e613-schema-derived-typed-item-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e614-typed-object-state-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e615-typed-object-slot-role-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e616-object-frame-slot-bias-scratch80-replay-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": "119dd41a57573fe408ee1208a9159cafca9a45c7050bd00274571938898a8539", + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e616-object-frame-slot-bias-scratch80-replay-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": "119dd41a57573fe408ee1208a9159cafca9a45c7050bd00274571938898a8539", + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e620-required-slot-coverage-scratch800-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e620-required-slot-coverage-scratch800-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e620-required-slot-coverage-scratch800-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e621-coverage-aware-closure-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e621-coverage-aware-closure-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e621-coverage-aware-closure-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e622-coverage-closure-trace-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e622-coverage-closure-trace-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e626-required-slot-margin-decode-weight-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e626-required-slot-margin-decode-weight-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e626-required-slot-margin-decode-weight-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e628-required-slot-margin-frame-depth-exclusion-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e628-required-slot-margin-frame-depth-exclusion-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e628-required-slot-margin-frame-depth-exclusion-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e630-prompt-owned-closure-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e630-prompt-owned-closure-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e631-frame-aware-owner-escape-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e631-frame-aware-owner-escape-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e632-input-role-assignment-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e633-input-active-role-routing-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e633-input-active-role-routing-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e633-input-active-role-routing-r2-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e634-final-precontent-routing-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e635-property-compatible-coverage-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e635-property-compatible-coverage-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e636-modal-schema-reach-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e636-modal-schema-reach-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e637-nested-family-accounting-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e637-nested-family-accounting-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e638-root-slot-coverage-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e639-root-sibling-coverage-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e639-root-sibling-coverage-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e640-root-slot-references-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e641-role-plan-completion-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e642-root-only-role-plans-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e643-bound-role-plans-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e644-role-obligation-margin-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e645-root-binding-w4-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e645-root-binding-w8-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e645-root-sibling-coverage-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e645-root-sibling-coverage-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e646-complete-root-reachability-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e646-root-slot-references-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e647-role-plan-completion-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e647-root-abstention-trace-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e647-root-abstention-trace-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e648-dynamic-literal-root-probe-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e648-root-only-role-plans-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e649-bound-role-plans-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e649-refresh-action-role-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e650-planned-family-role-binding-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e650-role-obligation-margin-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e651-schema-enum-literal-margin-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e652-value-text-role-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e653-nested-role-ownership-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e653-nested-role-ownership-r1-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e654-nested-role-enum-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e655-direct-role-slot-ownership-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e656-repeated-slot-role-ownership-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e657-combined-role-ownership-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e658-property-role-ownership-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e659-property-role-guard-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e666-schema-enum-finalize-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e667-nested-typed-array-owner-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e668-typed-array-role-wrapper-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e669-nested-array-schema-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e670-nested-array-role-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e671-nested-role-owner-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e672-schema-owned-array-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e673-schema-ref-reachability-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e674-semantic-role-alias-contract-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e675-schema-open-visibility-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e676-joint-role-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e678-joint-role-provenance-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e679-joint-role-smoke-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e680-display-role-aliases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e681-maximal-role-subsets-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e682-positional-role-binding-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e683-positional-role-heldout-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e684-singular-plural-family-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e685-numbered-tab-carriers-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e686-per-row-trace-budget-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e687-trace-record-identity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e688-bound-wrapper-roles-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e689-planned-child-carriers-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e690-bound-transitive-wrappers-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e691-structured-content-schema-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e692-fixed-enum-finalize-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e693-unique-group-plan-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e694-open-enum-finalize-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e695-role-property-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e696-role-literal-fallback-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e697-required-parent-order-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e698-likeness-abstention-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e699-role-capacity-revisit-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e700-namespace-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e701-schema-aware-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e701-schema-aware-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e701-schema-aware-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e701-schema-aware-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e701-schema-aware-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e702-joint-role-cardinality-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e702-joint-role-cardinality-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e702-joint-role-cardinality-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e702-joint-role-cardinality-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e702-joint-role-cardinality-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e703-enum-safe-repeated-slots-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e703-enum-safe-repeated-slots-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e703-enum-safe-repeated-slots-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e703-enum-safe-repeated-slots-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e703-enum-safe-repeated-slots-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e704-schema-value-weight-sweep-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e704-schema-value-weight-sweep-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e704-schema-value-weight-sweep-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e704-schema-value-weight-sweep-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e704-schema-value-weight-sweep-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e705-root-margin-routing-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e706-bounded-slot-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e706-bounded-slot-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e706-bounded-slot-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e706-bounded-slot-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e706-bounded-slot-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e707-carrier-root-reference-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e707-carrier-root-reference-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e707-carrier-root-reference-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e707-carrier-root-reference-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e707-carrier-root-reference-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e708-carrier-reference-obligation-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e708-carrier-reference-obligation-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e708-carrier-reference-obligation-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e708-carrier-reference-obligation-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e708-carrier-reference-obligation-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e709-final-schema-value-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e709-final-schema-value-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e709-final-schema-value-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e709-final-schema-value-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e709-final-schema-value-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e710-role-binding-negative-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e710-role-binding-negative-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e710-role-binding-negative-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e710-role-binding-negative-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e710-role-binding-negative-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e711-planned-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e711-planned-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e711-planned-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e711-planned-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e711-planned-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e712-component-count-phrases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e712-component-count-phrases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e712-component-count-phrases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e712-component-count-phrases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e712-component-count-phrases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e758-standalone-marker-sibling-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e759-broader-rico-prefix-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 9, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e760-e762-local-sampling-marker-uniqueness-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 8, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e763-e764-symbol-only-heldout-fallback-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "fallback_interference", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 4, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.2, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e765-e780-schema-closed-decoder-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.2, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e781-e790-numeric-literal-planning-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.6, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e792-e802-schema-array-contract-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.6, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-sde0-01-decode-scaffolding-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-sde0-01-decode-scaffolding-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-sde0-01-decode-scaffolding-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-sde0-01-decode-scaffolding-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-sde0-01-decode-scaffolding-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 32, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 32, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last.pt", + "checkpoint_sha256": "15a95b415884f59ffafea3e69826a41534b15afb72aeccecb85cd2099108f3b9", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last.pt", + "checkpoint_sha256": "15a95b415884f59ffafea3e69826a41534b15afb72aeccecb85cd2099108f3b9", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e175-retrieval-8step/checkpoints/last.pt", + "checkpoint_sha256": "ef8497ea80cd413e0c0e47f2d317ec6a331a8353ea94a11a16fd2feb682f0a34", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e175-retrieval-8step/checkpoints/last.pt", + "checkpoint_sha256": "ef8497ea80cd413e0c0e47f2d317ec6a331a8353ea94a11a16fd2feb682f0a34", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e176-broad-corpus-8step/checkpoints/last.pt", + "checkpoint_sha256": "996a5a43220b8c0216d389164b9056d8817e1fa9523ed88ff0560f6cee6c4ed1", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e176-broad-corpus-8step/checkpoints/last.pt", + "checkpoint_sha256": "996a5a43220b8c0216d389164b9056d8817e1fa9523ed88ff0560f6cee6c4ed1", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "453208f977d95b1c97d1c776558ddfa8fae000151fbf99a4ef3440f443deeda0", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "453208f977d95b1c97d1c776558ddfa8fae000151fbf99a4ef3440f443deeda0", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3bf8b8d25309818e438fd2814796edf3bd5825b454fd900ffdb80f2989615e4f", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3bf8b8d25309818e438fd2814796edf3bd5825b454fd900ffdb80f2989615e4f", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "6674261fe499272fd9c108fdb9985ae33029248ac4611f80b4119941e472c0ea", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "6674261fe499272fd9c108fdb9985ae33029248ac4611f80b4119941e472c0ea", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3e1463436a28fa1d12ef28451c5aa90d186d0254619e8d50080b19a4c9ea2c58", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3e1463436a28fa1d12ef28451c5aa90d186d0254619e8d50080b19a4c9ea2c58", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last.pt", + "checkpoint_sha256": "8f700fa730cda5f67b6184be3a54fc36a754ff2a84fee8038b3e85b3dca3eb37", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last.pt", + "checkpoint_sha256": "8f700fa730cda5f67b6184be3a54fc36a754ff2a84fee8038b3e85b3dca3eb37", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e219-schema-normalized-32step/checkpoints/last.pt", + "checkpoint_sha256": "48ba5a7aaec6507a75b403d81603f3249ae5cb94b578217ee22071ed8d49e112", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e219-schema-normalized-32step/checkpoints/last.pt", + "checkpoint_sha256": "48ba5a7aaec6507a75b403d81603f3249ae5cb94b578217ee22071ed8d49e112", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c08fd319cd7b72e19990a63496dcccb5850282c11cfd9095817bd92a87848afb", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c08fd319cd7b72e19990a63496dcccb5850282c11cfd9095817bd92a87848afb", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "da1cd4d759fb11c056d69f701cdf8903d2dd1f9d33c98847b25c758b4569c9d8", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "da1cd4d759fb11c056d69f701cdf8903d2dd1f9d33c98847b25c758b4569c9d8", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6bcdb828756916d436daae07c96874d12ffcf08ff211fcec923763c433ff2ed5", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6bcdb828756916d436daae07c96874d12ffcf08ff211fcec923763c433ff2ed5", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "33a8511d8dfaeb50072f42e29b8c5fb23d30b47012e98586491291125fafdfcd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "33a8511d8dfaeb50072f42e29b8c5fb23d30b47012e98586491291125fafdfcd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7be4ccaa98f4aa071843219ebe11ad0823c6323a1a8a49064aa43869eca9cf10", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7be4ccaa98f4aa071843219ebe11ad0823c6323a1a8a49064aa43869eca9cf10", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7ef101ae1c1859c9eb6561f47e37b6dada9636a346667f1270800222b5362147", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7ef101ae1c1859c9eb6561f47e37b6dada9636a346667f1270800222b5362147", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "1e27051441954b7f1f5c2fbcdf38b7d02ebac53c97452a24239b0a4b8a45187b", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "1e27051441954b7f1f5c2fbcdf38b7d02ebac53c97452a24239b0a4b8a45187b", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "10b05f6f153a101bac1b22979e010cee506f3c313fd34f42f6ba245d7783ff7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "10b05f6f153a101bac1b22979e010cee506f3c313fd34f42f6ba245d7783ff7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "bda7e7f35134f3f9a6ca1d84b7d970250f2246951c71ae7bf4b563d023fbb7bf", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "bda7e7f35134f3f9a6ca1d84b7d970250f2246951c71ae7bf4b563d023fbb7bf", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6d485bae0c32092454794f41af96abf875465f1fd82f9ff12c67c868d596609a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6d485bae0c32092454794f41af96abf875465f1fd82f9ff12c67c868d596609a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "2755ab5f7b700dcc5cda01980df931dcae311cc41abc181a9ba49e346e99986e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "2755ab5f7b700dcc5cda01980df931dcae311cc41abc181a9ba49e346e99986e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "91f22e60151c7506dd7d64c3d427f4bc8466b8f8361e29ede9d884a83c79aaf7", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "91f22e60151c7506dd7d64c3d427f4bc8466b8f8361e29ede9d884a83c79aaf7", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "df159e8ed49ad1107fca9017ce369435e991930a625ef96193289084df34f3dc", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "df159e8ed49ad1107fca9017ce369435e991930a625ef96193289084df34f3dc", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c5ae47eed8b36ba070b76734cb503385a5a61791ebcbf2e2635bf86148c56a39", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c5ae47eed8b36ba070b76734cb503385a5a61791ebcbf2e2635bf86148c56a39", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "777924cad2660579359fd8b1dba1961c83ac54f0c8fb7bb8e12e483eceeb1f2c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "777924cad2660579359fd8b1dba1961c83ac54f0c8fb7bb8e12e483eceeb1f2c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "e31fcfd2b03e6740ead7ce691ebf373e16de908b366b190117f25c04cf38161c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "e31fcfd2b03e6740ead7ce691ebf373e16de908b366b190117f25c04cf38161c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "95d8868d59df45dd7c22a27d09b7205ce375c9ef97aaa1d8070e9728cfea7237", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "95d8868d59df45dd7c22a27d09b7205ce375c9ef97aaa1d8070e9728cfea7237", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7db15eff2a3a73e6372d912222667208b2bd8fc8493beb7638565e6385a792d4", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7db15eff2a3a73e6372d912222667208b2bd8fc8493beb7638565e6385a792d4", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fd15d55eb8451ce089ca0299f8cbe5962a07f6fb71790dfd51c5b14c30309f4b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fd15d55eb8451ce089ca0299f8cbe5962a07f6fb71790dfd51c5b14c30309f4b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3b5281385368b5a575b0766c2c8a8407a45527535cb055751c8fd74222d624d7", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3b5281385368b5a575b0766c2c8a8407a45527535cb055751c8fd74222d624d7", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fa24f6b2f5c235514f0f01e6fff567d24054a59974e81593423d4ad2e636f075", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fa24f6b2f5c235514f0f01e6fff567d24054a59974e81593423d4ad2e636f075", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f423835433c85560049881bfa8186a1c46715ecc8f6e7cad8aa24b9c65915674", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f423835433c85560049881bfa8186a1c46715ecc8f6e7cad8aa24b9c65915674", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "a21e92839eea9456397c51644a22e5fcf2314356f103f584b9f5194a6f175fe8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "a21e92839eea9456397c51644a22e5fcf2314356f103f584b9f5194a6f175fe8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "00c6a23f363be27a1013b279411703b3872e4444f3c59baec15a67d2dd968ba8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "00c6a23f363be27a1013b279411703b3872e4444f3c59baec15a67d2dd968ba8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "ed9f257bc9463368c07ea54179dbddae69ca41ab2085f7cb8aed75d7b6aadc9e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "ed9f257bc9463368c07ea54179dbddae69ca41ab2085f7cb8aed75d7b6aadc9e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/slm294_tiny_baseline/checkpoints/last.pt", + "checkpoint_sha256": "1eb2b2df1fcc8de1ede419bf70ab64d3ff4e21d355e3d6d30895f4003ce109fc", + "suite": "smoke", + "classification": "unmeasured", + "n": 0, + "decode_timeout_count": 0, + "fallback_count": null, + "decode_canvas_cap": 256, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/slm294_tiny_baseline/checkpoints/last.pt", + "checkpoint_sha256": "1eb2b2df1fcc8de1ede419bf70ab64d3ff4e21d355e3d6d30895f4003ce109fc", + "suite": "smoke", + "classification": "unmeasured", + "n": 0, + "decode_timeout_count": 0, + "fallback_count": null, + "decode_canvas_cap": 256, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 14, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 15, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 15, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 1500, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 1, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": "119dd41a57573fe408ee1208a9159cafca9a45c7050bd00274571938898a8539", + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": "119dd41a57573fe408ee1208a9159cafca9a45c7050bd00274571938898a8539", + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 9, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 8, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "fallback_interference", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 4, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.2, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.2, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.6, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.6, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + } + ], + "hash_pins": { + "ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "actual_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "scoreboard_recorded_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "matches_scoreboard_record": true + }, + "lexer_ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "actual_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "scoreboard_recorded_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "matches_scoreboard_record": true + } + }, + "summary": { + "n_local_checkpoints": 216, + "n_roster_rows": 115, + "n_committed_sha_verified": 1, + "n_committed_sha_mismatch": 0, + "n_unverifiable": 305, + "scoreboard_classes": { + "fallback_interference": 2, + "model_behavior": 1134, + "runtime_timeout_interference": 10, + "unmeasured": 4 + } + } + }, + "sweep": { + "n_cells": 12, + "pairs": [ + { + "checkpoint": "lexer_ltr2", + "record_id": "smoke_button_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 2091.33, + "elapsed_ms_10x": 1683.37 + }, + { + "checkpoint": "lexer_ltr2", + "record_id": "smoke_callout_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 1643.82, + "elapsed_ms_10x": 1693.5 + }, + { + "checkpoint": "lexer_ltr2", + "record_id": "smoke_hero_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 1901.12, + "elapsed_ms_10x": 1589.12 + }, + { + "checkpoint": "ltr2", + "record_id": "smoke_button_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 1276.21, + "elapsed_ms_10x": 1519.6 + }, + { + "checkpoint": "ltr2", + "record_id": "smoke_callout_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 1196.94, + "elapsed_ms_10x": 1842.18 + }, + { + "checkpoint": "ltr2", + "record_id": "smoke_hero_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 1217.16, + "elapsed_ms_10x": 2402.0 + } + ], + "outcome_counts": { + "baseline": { + "model_valid": 0, + "model_invalid": 0, + "model_abstain": 0, + "runtime_timeout": 0, + "fallback_output": 0, + "harness_error": 0 + }, + "budget10x": { + "model_valid": 0, + "model_invalid": 0, + "model_abstain": 0, + "runtime_timeout": 0, + "fallback_output": 0, + "harness_error": 0 + } + }, + "cells": [ + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 2091.33, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_button_01", + "record_sha256": "2caa4109b55685eda55860df59afb5d16b51ce55399ae1411416709f8ffe1e82", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1643.82, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_callout_01", + "record_sha256": "ddab24fb33b82c665cd22bbfffd7895b4d425a7fade5fd0bb6fadb79effe91b3", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1901.12, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_hero_01", + "record_sha256": "b44a979b99563d2c5744ba63612f05c6231cbb064593bf5da021f48fcb27605e", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1683.37, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_button_01", + "record_sha256": "2caa4109b55685eda55860df59afb5d16b51ce55399ae1411416709f8ffe1e82", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1693.5, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_callout_01", + "record_sha256": "ddab24fb33b82c665cd22bbfffd7895b4d425a7fade5fd0bb6fadb79effe91b3", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1589.12, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_hero_01", + "record_sha256": "b44a979b99563d2c5744ba63612f05c6231cbb064593bf5da021f48fcb27605e", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1276.21, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_button_01", + "record_sha256": "2caa4109b55685eda55860df59afb5d16b51ce55399ae1411416709f8ffe1e82", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1196.94, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_callout_01", + "record_sha256": "ddab24fb33b82c665cd22bbfffd7895b4d425a7fade5fd0bb6fadb79effe91b3", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1217.16, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_hero_01", + "record_sha256": "b44a979b99563d2c5744ba63612f05c6231cbb064593bf5da021f48fcb27605e", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1519.6, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_button_01", + "record_sha256": "2caa4109b55685eda55860df59afb5d16b51ce55399ae1411416709f8ffe1e82", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1842.18, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_callout_01", + "record_sha256": "ddab24fb33b82c665cd22bbfffd7895b4d425a7fade5fd0bb6fadb79effe91b3", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 2402.0, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_hero_01", + "record_sha256": "b44a979b99563d2c5744ba63612f05c6231cbb064593bf5da021f48fcb27605e", + "seed": 0, + "outcome_record": null + } + ] + }, + "disposition": { + "annotations": [ + { + "scoreboard": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-e497-current-main-playground-provenance-smoke-20260718.json", + "annotation": "runtime/harness artifact: 1/3 decode timeouts at canvas cap None; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "runtime/harness artifact: 1/3 decode timeouts at canvas cap None; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-e763-e764-symbol-only-heldout-fallback-20260722.json", + "annotation": "fallback interference: fallback_count=4; fallback outputs never classify as model success" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "fallback interference: fallback_count=4; fallback outputs never classify as model success" + } + ], + "counts": { + "runtime_timeout_interference": 10, + "fallback_interference": 2, + "unmeasured": 4 + }, + "sweep_verdict": "not_rerunnable: 6/6 paired cells blocked by the output-contract gate (checkpoint v0 vs required symbol_only/v2, no migration path); the 10x-budget flip question is UNANSWERED by re-decode \u2014 census + taxonomy carry the audit", + "recommendation": { + "powered_rerun": "at most one: retrain the remediated recipe from symbol-only targets (output contract v2 \u2014 the v0 checkpoints are blocked by require_current_output_contract with no migration path), then a preregistered full-smoke re-eval at recorded vs 10x decode budget with per-record decode_outcome fields (now emitted by eval_runner), n\u226516 smoke+fixture records for Wilson resolution", + "rationale": "census localizes all timeout interference to the two remediated checkpoints; no other hash-verifiable checkpoint carries a timeout-flagged scoreboard, and re-decode of the v0 checkpoints under current code is impossible" + } + }, + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "cc2521119d6d89d4663131a7001ce45897ca8cdd", + "code_dirty": true, + "components": { + "harness.experiments.slm303_decode_budget_audit": "v1", + "harness.model_build.eval": "v51" + }, + "stamped_at": "2026-07-24T19:08:59.985574+00:00" + } +} diff --git a/docs/design/iter-slm303-decode-budget-audit-20260724.md b/docs/design/iter-slm303-decode-budget-audit-20260724.md new file mode 100644 index 000000000..1726171cf --- /dev/null +++ b/docs/design/iter-slm303-decode-budget-audit-20260724.md @@ -0,0 +1,54 @@ +# SLM-303 (LAR1-05): decode-budget audit — census, 10x sweep, disposition + +- generated_at: `2026-07-24T19:08:59Z` +- claim_class: `diagnostic` (diagnostic; not ship evidence) +- taxonomy: `model_valid, model_invalid, model_abstain, runtime_timeout, fallback_output, harness_error` +- budget evidence: inferred from recorded eval_smoke.json: all 3 smoke rows share latency_ms=3333.79 (one chunk of 3 killed at ~10001ms → 10s timeout); the original CLI flag was not persisted (config=null), disclosed as inferred + +## Census + +- local checkpoints enumerated: 216 (roster rows: 115) +- committed SHA verified: 1; mismatch: 0; unverifiable (no committed SHA): 305 +- scoreboard classes: `{"fallback_interference": 2, "model_behavior": 1134, "runtime_timeout_interference": 10, "unmeasured": 4}` + +### Hash pins (sweep checkpoints) + +| key | path | sha256 | matches scoreboard record | +| --- | --- | --- | --- | +| ltr2 | `outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt` | `653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a` | True | +| lexer_ltr2 | `outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt` | `4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c` | True | + +## Budget sweep (preregistered, paired) + +| checkpoint | record | baseline outcome | 10x outcome | baseline ms | 10x ms | +| --- | --- | --- | --- | --- | --- | +| lexer_ltr2 | smoke_button_01 | not_rerunnable | not_rerunnable | 2091.33 | 1683.37 | +| lexer_ltr2 | smoke_callout_01 | not_rerunnable | not_rerunnable | 1643.82 | 1693.5 | +| lexer_ltr2 | smoke_hero_01 | not_rerunnable | not_rerunnable | 1901.12 | 1589.12 | +| ltr2 | smoke_button_01 | not_rerunnable | not_rerunnable | 1276.21 | 1519.6 | +| ltr2 | smoke_callout_01 | not_rerunnable | not_rerunnable | 1196.94 | 1842.18 | +| ltr2 | smoke_hero_01 | not_rerunnable | not_rerunnable | 1217.16 | 2402.0 | + +- outcome counts (baseline): `{"model_valid": 0, "model_invalid": 0, "model_abstain": 0, "runtime_timeout": 0, "fallback_output": 0, "harness_error": 0}` +- outcome counts (10x): `{"model_valid": 0, "model_invalid": 0, "model_abstain": 0, "runtime_timeout": 0, "fallback_output": 0, "harness_error": 0}` + +## Disposition (append-only annotations) + +- `outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict +- `outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval_smoke.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict +- `outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict +- `outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-e497-current-main-playground-provenance-smoke-20260718.json`: runtime/harness artifact: 1/3 decode timeouts at canvas cap None; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: runtime/harness artifact: 1/3 decode timeouts at canvas cap None; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-e763-e764-symbol-only-heldout-fallback-20260722.json`: fallback interference: fallback_count=4; fallback outputs never classify as model success +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: fallback interference: fallback_count=4; fallback outputs never classify as model success + +- sweep verdict: not_rerunnable: 6/6 paired cells blocked by the output-contract gate (checkpoint v0 vs required symbol_only/v2, no migration path); the 10x-budget flip question is UNANSWERED by re-decode — census + taxonomy carry the audit +- powered-rerun recommendation: at most one: retrain the remediated recipe from symbol-only targets (output contract v2 — the v0 checkpoints are blocked by require_current_output_contract with no migration path), then a preregistered full-smoke re-eval at recorded vs 10x decode budget with per-record decode_outcome fields (now emitted by eval_runner), n≥16 smoke+fixture records for Wilson resolution + - rationale: census localizes all timeout interference to the two remediated checkpoints; no other hash-verifiable checkpoint carries a timeout-flagged scoreboard, and re-decode of the v0 checkpoints under current code is impossible + +Historical iter docs and MODEL_CARD rows are untouched; these annotations are additive. diff --git a/scripts/run_slm303_decode_budget_audit.py b/scripts/run_slm303_decode_budget_audit.py new file mode 100644 index 000000000..eaf8e9b72 --- /dev/null +++ b/scripts/run_slm303_decode_budget_audit.py @@ -0,0 +1,917 @@ +#!/usr/bin/env python3 +"""SLM-303 (LAR1-05): decode-budget audit — census, budget sweep, disposition. + +Question: which historical zero / interference-flagged scoreboards are model +behavior, and which are runtime/harness budget artifacts (decode timeouts, +fallbacks)? Answered three ways: + +1. ``census`` — enumerate every local ``outputs/runs/*/checkpoints/*.pt`` plus + every MODEL_CARD roster row; verify committed SHA fragments + (prefix…suffix) against actual sha256 of local bytes; group every local + ``eval_*.json`` scoreboard (and committed docs/design iter JSONs for + historical context) by checkpoint; classify each scoreboard with the + SLM-303 decode-outcome taxonomy (``unmeasured`` when fallback_count is + null). Hash-pins the two remediated checkpoints that carry the only + nonzero-timeout scoreboards. +2. ``sweep-cell`` / ``report`` — PREREGISTERED budget sweep: same checkpoint, + same 3 smoke records (test_seeds smoke ids — the original suite directory + is empty, so the suite is rebuilt from test_seeds; disclosed), same seed, + checkpoint-declared decode policy; arms differ ONLY in budget: baseline + (recorded canvas cap 256/128, inferred 10s decode timeout from the recorded + 3×3333.79ms chunk latency) vs 10x (100s, 2560/1280). Each + (checkpoint, arm, record) cell runs in its own hard-killed worker process + so every command stays under the run cap; a killed cell is ``decode_timeout`` + evidence, never a fabricated result. +3. ``report`` — disposition: append-only historical annotations + a small + powered-rerun recommendation, written to + ``docs/design/iter-slm303-decode-budget-audit-20260724.{json,md}``. + +Example: + python -m scripts.run_slm303_decode_budget_audit census + python -m scripts.run_slm303_decode_budget_audit sweep-cell \ + --checkpoint ltr2 --arm baseline --record-id smoke_hero_01 + python -m scripts.run_slm303_decode_budget_audit report +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from slm_training.harnesses.model_build.decode_outcome import ( + DECODE_OUTCOMES, + DecodeOutcomeRecordV1, + classify_decode_outcome, + outcome_counts, +) +from slm_training.versioning import build_version_stamp + +COMPONENT = "harness.experiments.slm303_decode_budget_audit" +EVAL_COMPONENT = "harness.model_build.eval" + +RUNS_ROOT = Path("outputs/runs") +MODEL_CARD = Path("docs/MODEL_CARD.md") +DESIGN_DIR = Path("docs/design") +TEST_SEEDS = Path("src/slm_training/resources/test_seeds.jsonl") + +AUDIT_DIR = RUNS_ROOT / "slm303_decode_budget_audit" +CENSUS_JSON = AUDIT_DIR / "census.json" +CELLS_DIR = AUDIT_DIR / "cells" + +DEFAULT_JSON_OUT = DESIGN_DIR / "iter-slm303-decode-budget-audit-20260724.json" +DEFAULT_MD_OUT = DESIGN_DIR / "iter-slm303-decode-budget-audit-20260724.md" + +# The only two checkpoints with ACTUAL nonzero-timeout scoreboards. Canvas +# caps and timeout evidence come from their recorded eval_smoke.json. +SWEEP_CHECKPOINTS: dict[str, dict[str, Any]] = { + "ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "scoreboard": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "recorded_canvas_cap": 256, + "recorded_decode_timeout_count": 3, + }, + "lexer_ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "scoreboard": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "recorded_canvas_cap": 128, + "recorded_decode_timeout_count": 3, + }, +} + +# The recorded timeout scoreboards show latency_ms = 3333.79 on all 3 smoke +# records: one chunk of 3 killed at ~10001ms → decode_timeout_seconds = 10. +BASELINE_TIMEOUT_S = 10.0 +BASELINE_TIMEOUT_EVIDENCE = ( + "inferred from recorded eval_smoke.json: all 3 smoke rows share " + "latency_ms=3333.79 (one chunk of 3 killed at ~10001ms → 10s timeout); " + "the original CLI flag was not persisted (config=null), disclosed as inferred" +) +SWEEP_BUDGET_MULTIPLIER = 10 + +SMOKE_RECORD_IDS = ("smoke_hero_01", "smoke_button_01", "smoke_callout_01") + +_SHA_FRAGMENT_RE = re.compile(r"\b([0-9a-f]{6,})…([0-9a-f]{4,})\b") +_SHA_FULL_RE = re.compile(r"\b[0-9a-f]{64}\b") +_CHECKPOINT_PATH_RE = re.compile(r"outputs/runs/\S*?checkpoints/[\w.\-]+\.pt") + + +# --------------------------------------------------------------------------- +# Pure helpers (unit-tested) +# --------------------------------------------------------------------------- + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + for block in iter(lambda: fh.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def sha_matches_committed(committed: str | None, actual: str) -> bool | None: + """Verify a committed SHA reference against actual bytes. + + ``committed`` may be a full 64-hex digest or a ``prefix…suffix`` fragment + (as used in MODEL_CARD roster rows). Returns None when no committed + reference exists (unverifiable, never a pass). + """ + if not committed: + return None + committed = committed.strip().strip("`") + if _SHA_FULL_RE.fullmatch(committed): + return committed == actual + match = _SHA_FRAGMENT_RE.search(committed) + if not match: + return None + prefix, suffix = match.groups() + return actual.startswith(prefix) and actual.endswith(suffix) + + +def classify_scoreboard(scoreboard: dict[str, Any]) -> dict[str, Any]: + """Classify one scoreboard dict with the decode-outcome taxonomy. + + Scoreboard-level view: ``unmeasured`` when fallback telemetry is absent + (fallback_count null), ``runtime_timeout_interference`` when decode + timeouts fired, ``fallback_interference`` when fallbacks fired, else + ``model_behavior`` — a clean zero is a model verdict, not a budget + artifact. Per-record rows are classified when details[] exists. + """ + fallback_count = scoreboard.get("fallback_count") + timeout_count = scoreboard.get("decode_timeout_count") + n = scoreboard.get("n") + row_outcomes: list[dict[str, Any]] = [] + for row in scoreboard.get("details") or []: + if not isinstance(row, dict) or "id" not in row: + continue + row_outcomes.append( + { + "id": row.get("id"), + "outcome": classify_decode_outcome( + parse_ok=row.get("parse_ok"), + error=row.get("error"), + fallback_counters=int(row.get("fallback_count") or 0), + timed_out=False, # historical rows carry no timeout fact + abstained=not str(row.get("prediction") or "").strip(), + harness_exception=False, + ), + "note": ( + "historical row: no per-record timeout/fallback fields; " + "timeout attribution comes from the suite-level count" + ), + } + ) + if fallback_count is None: + classification = "unmeasured" + elif timeout_count: + classification = "runtime_timeout_interference" + elif fallback_count: + classification = "fallback_interference" + else: + classification = "model_behavior" + return { + "classification": classification, + "n": n, + "decode_timeout_count": timeout_count, + "fallback_count": fallback_count, + "decode_canvas_cap": scoreboard.get("decode_canvas_cap"), + "parse_rate": scoreboard.get("parse_rate"), + "meaningful_program_rate": scoreboard.get("meaningful_program_rate"), + "row_outcomes": row_outcomes, + } + + +def build_sweep_arms(recorded_canvas_cap: int) -> dict[str, dict[str, Any]]: + """Preregistered arms; they differ ONLY in budget fields.""" + return { + "baseline": { + "timeout_s": BASELINE_TIMEOUT_S, + "canvas_cap": recorded_canvas_cap, + "timeout_evidence": BASELINE_TIMEOUT_EVIDENCE, + }, + "budget10x": { + "timeout_s": BASELINE_TIMEOUT_S * SWEEP_BUDGET_MULTIPLIER, + "canvas_cap": recorded_canvas_cap * SWEEP_BUDGET_MULTIPLIER, + "timeout_evidence": "10x the preregistered baseline budget", + }, + } + + +def pair_cells( + cells: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Pair baseline vs budget10x cells per (checkpoint, record); assert the + arms differ only in budget fields (paired-design check).""" + pairs: list[dict[str, Any]] = [] + by_key: dict[tuple[str, str], dict[str, Any]] = {} + for cell in cells: + by_key.setdefault((cell["checkpoint"], cell["record_id"]), {})[ + cell["arm"] + ] = cell + for (checkpoint, record_id), arms in sorted(by_key.items()): + base = arms.get("baseline") + wide = arms.get("budget10x") + if base is None or wide is None: + pairs.append( + { + "checkpoint": checkpoint, + "record_id": record_id, + "status": "unpaired", + "arms_present": sorted(arms), + } + ) + continue + base_budget = base["budget"] + wide_budget = wide["budget"] + non_budget_mismatch = [ + field + for field in ("checkpoint_sha256", "seed", "record_sha256") + if base.get(field) != wide.get(field) + ] + pairs.append( + { + "checkpoint": checkpoint, + "record_id": record_id, + "status": "paired" if not non_budget_mismatch else "pair_violation", + "pair_violation_fields": non_budget_mismatch, + "budget_baseline": base_budget, + "budget_10x": wide_budget, + "outcome_baseline": base["outcome"], + "outcome_10x": wide["outcome"], + "cell_status_baseline": base.get("status"), + "cell_status_10x": wide.get("status"), + "parse_ok_baseline": base.get("parse_ok"), + "parse_ok_10x": wide.get("parse_ok"), + "elapsed_ms_baseline": base.get("elapsed_ms"), + "elapsed_ms_10x": wide.get("elapsed_ms"), + } + ) + return pairs + + +# --------------------------------------------------------------------------- +# Census +# --------------------------------------------------------------------------- + + +def _roster_rows(text: str) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for line in text.splitlines(): + for path in _CHECKPOINT_PATH_RE.findall(line): + fragment = _SHA_FRAGMENT_RE.search(line) + full = _SHA_FULL_RE.search(line) + committed = None + if full: + committed = full.group(0) + elif fragment: + committed = f"{fragment.group(1)}…{fragment.group(2)}" + rows.append({"path": path, "committed_sha": committed}) + return rows + + +def _iter_scoreboards() -> list[tuple[Path, dict[str, Any]]]: + """Local eval_*.json scoreboards (path, payload).""" + out: list[tuple[Path, dict[str, Any]]] = [] + for path in sorted(RUNS_ROOT.glob("*/eval*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception: # noqa: BLE001 - unreadable artifact is skipped, noted + continue + if isinstance(payload, dict) and ( + "decode_timeout_count" in payload or "fallback_count" in payload + ): + out.append((path, payload)) + return out + + +def _design_scoreboards() -> list[tuple[Path, dict[str, Any]]]: + """Committed docs/design iter JSONs carrying scoreboard-shaped dicts.""" + out: list[tuple[Path, dict[str, Any]]] = [] + + def _walk(node: Any, path: Path) -> None: + if isinstance(node, dict): + if "decode_timeout_count" in node and "fallback_count" in node: + out.append((path, node)) + return + for value in node.values(): + _walk(value, path) + elif isinstance(node, list): + for value in node: + _walk(value, path) + + for path in sorted(DESIGN_DIR.glob("iter*.json")): + try: + _walk(json.loads(path.read_text(encoding="utf-8")), path) + except Exception: # noqa: BLE001 + continue + return out + + +def run_census(*, out_path: Path = CENSUS_JSON) -> dict[str, Any]: + roster = _roster_rows(MODEL_CARD.read_text(encoding="utf-8")) + roster_by_path = {row["path"]: row for row in roster} + + local_checkpoints = sorted(RUNS_ROOT.glob("*/checkpoints/*.pt")) + checkpoints: list[dict[str, Any]] = [] + seen: set[str] = set() + for path in local_checkpoints: + rel = str(path) + seen.add(rel) + actual = sha256_file(path) + committed = (roster_by_path.get(rel) or {}).get("committed_sha") + checkpoints.append( + { + "path": rel, + "exists_locally": True, + "in_roster": rel in roster_by_path, + "committed_sha": committed, + "actual_sha256": actual, + "committed_sha_match": sha_matches_committed(committed, actual), + } + ) + for rel, row in sorted(roster_by_path.items()): + if rel not in seen: + checkpoints.append( + { + "path": rel, + "exists_locally": False, + "in_roster": True, + "committed_sha": row["committed_sha"], + "actual_sha256": None, + "committed_sha_match": None, + } + ) + + scoreboards: list[dict[str, Any]] = [] + for path, payload in _iter_scoreboards(): + entry = { + "scoreboard_path": str(path), + "source": "local_outputs", + "checkpoint": payload.get("checkpoint"), + "checkpoint_sha256": payload.get("checkpoint_sha256"), + "suite": payload.get("suite"), + **classify_scoreboard(payload), + } + scoreboards.append(entry) + for path, node in _design_scoreboards(): + scoreboards.append( + { + "scoreboard_path": str(path), + "source": "docs_design_historical", + "checkpoint": node.get("checkpoint"), + "checkpoint_sha256": node.get("checkpoint_sha256"), + "suite": node.get("suite"), + **classify_scoreboard(node), + } + ) + + hash_pins: dict[str, Any] = {} + for key, spec in SWEEP_CHECKPOINTS.items(): + path = Path(spec["path"]) + pin: dict[str, Any] = {"path": spec["path"], "exists_locally": path.exists()} + if path.exists(): + actual = sha256_file(path) + pin["actual_sha256"] = actual + recorded = None + sb_path = Path(spec["scoreboard"]) + if sb_path.exists(): + recorded = json.loads(sb_path.read_text(encoding="utf-8")).get( + "checkpoint_sha256" + ) + pin["scoreboard_recorded_sha256"] = recorded + pin["matches_scoreboard_record"] = ( + None if recorded is None else actual == recorded + ) + hash_pins[key] = pin + + census = { + "schema": "slm303_decode_census/v1", + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "taxonomy": list(DECODE_OUTCOMES), + "checkpoints": checkpoints, + "scoreboards": scoreboards, + "hash_pins": hash_pins, + "summary": { + "n_local_checkpoints": len(local_checkpoints), + "n_roster_rows": len(roster), + "n_committed_sha_verified": sum( + 1 for c in checkpoints if c["committed_sha_match"] is True + ), + "n_committed_sha_mismatch": sum( + 1 for c in checkpoints if c["committed_sha_match"] is False + ), + "n_unverifiable": sum( + 1 for c in checkpoints if c["committed_sha_match"] is None + ), + "scoreboard_classes": dict( + sorted( + { + cls: sum( + 1 for s in scoreboards if s["classification"] == cls + ) + for cls in { + s["classification"] for s in scoreboards + } + }.items() + ) + ), + }, + } + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(census, indent=2) + "\n", encoding="utf-8") + return census + + +# --------------------------------------------------------------------------- +# Sweep cells (hard-killed worker per (checkpoint, arm, record)) +# --------------------------------------------------------------------------- + + +def _load_smoke_records() -> dict[str, Any]: + from slm_training.dsl.schema import ExampleRecord + + records: dict[str, Any] = {} + with TEST_SEEDS.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + row = json.loads(line) + if row["id"] in SMOKE_RECORD_IDS: + records[row["id"]] = ExampleRecord( + id=row["id"], + prompt=row["prompt"], + openui=row["openui"], + placeholders=list(row.get("placeholders", [])), + split=row.get("split", "smoke"), + source=row.get("source", "smoke"), + ) + return records + + +def _cell_worker( + checkpoint_path: str, arm_budget: dict[str, Any], record_id: str, queue: Any +) -> None: + """Load the checkpoint fresh, decode one smoke record under the arm budget.""" + try: + from slm_training.harnesses.model_build.plugin import GenerationRequest + from slm_training.models.twotower import TwoTowerModel + + record = _load_smoke_records()[record_id] + model = TwoTowerModel.from_checkpoint(checkpoint_path, device="cpu") + model.config.grammar_ltr_max_tokens = int(arm_budget["canvas_cap"]) + request = GenerationRequest.from_record(record) + t0 = time.perf_counter() + pred = model.generate_batch_requests( + [request], max_len=int(arm_budget["canvas_cap"]) + )[0] + queue.put( + { + "status": "ok", + "elapsed_ms": round((time.perf_counter() - t0) * 1000.0, 2), + "prediction": pred, + } + ) + except Exception as exc: # noqa: BLE001 - decode failure is data + queue.put({"status": "harness_error", "error": str(exc)[:400]}) + + +def run_sweep_cell( + *, + checkpoint_key: str, + arm: str, + record_id: str, + out_dir: Path = CELLS_DIR, +) -> dict[str, Any]: + import multiprocessing as mp + + spec = SWEEP_CHECKPOINTS[checkpoint_key] + arms = build_sweep_arms(int(spec["recorded_canvas_cap"])) + budget = arms[arm] + checkpoint_path = spec["path"] + checkpoint_sha = sha256_file(Path(checkpoint_path)) + record = _load_smoke_records()[record_id] + record_sha = hashlib.sha256( + json.dumps(record.to_dict(), sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + ctx = mp.get_context("fork") + queue: Any = ctx.Queue() + proc = ctx.Process( + target=_cell_worker, args=(checkpoint_path, budget, record_id, queue) + ) + t0 = time.perf_counter() + proc.start() + proc.join(float(budget["timeout_s"])) + elapsed_ms = round((time.perf_counter() - t0) * 1000.0, 2) + if proc.is_alive(): + proc.terminate() + proc.join(30) + outcome = classify_decode_outcome( + parse_ok=None, timed_out=True, fallback_counters=0 + ) + cell = { + "status": "decode_timeout", + "outcome": outcome, + "stop_reason": "decode_timeout", + "elapsed_ms": elapsed_ms, + "prediction": None, + "parse_ok": None, + } + else: + result = ( + queue.get() + if not queue.empty() + else { + "status": "harness_error", + "error": f"worker exited without result (code {proc.exitcode})", + } + ) + if result["status"] == "ok": + pred = str(result["prediction"]) + from slm_training.dsl.parser import validate + + try: + validate(pred) + parse_ok: bool | None = True + error = None + except Exception as exc: # noqa: BLE001 - parse fact, never raised + parse_ok = False + error = str(exc)[:200] + outcome = classify_decode_outcome( + parse_ok=parse_ok, + error=error, + abstained=not pred.strip(), + fallback_counters=0, + timed_out=False, + ) + cell = { + "status": "ok", + "outcome": outcome, + "stop_reason": "empty_prediction" if not pred.strip() else "completed", + "elapsed_ms": result["elapsed_ms"], + "prediction": pred, + "parse_ok": parse_ok, + "error": error, + } + else: + error_text = str(result.get("error") or "") + not_rerunnable = "output contract" in error_text + cell = { + "status": "not_rerunnable" if not_rerunnable else "harness_error", + # No decode attempt happened → no outcome, never a guess. + "outcome": None if not_rerunnable else classify_decode_outcome( + parse_ok=None, harness_exception=True + ), + "stop_reason": "not_rerunnable" if not_rerunnable else "harness_error", + "elapsed_ms": elapsed_ms, + "prediction": None, + "parse_ok": None, + "error": result.get("error"), + } + + record_out = ( + DecodeOutcomeRecordV1( + request_id=record_id, + outcome=cell["outcome"], + stop_reason=cell["stop_reason"], + budget={ + "timeout_s": budget["timeout_s"], + "canvas_cap": budget["canvas_cap"], + "gen_steps": None, + "max_attempts": 1, + }, + elapsed_ms=cell["elapsed_ms"], + fallback_used=False, + detail=cell.get("error"), + ) + if cell["outcome"] is not None + else None + ) + budget_dict = { + "timeout_s": budget["timeout_s"], + "canvas_cap": budget["canvas_cap"], + "gen_steps": None, + "max_attempts": 1, + } + cell.update( + { + "schema": "slm303_sweep_cell/v1", + "checkpoint": checkpoint_key, + "checkpoint_path": checkpoint_path, + "checkpoint_sha256": checkpoint_sha, + "arm": arm, + "budget": budget_dict, + "record_id": record_id, + "record_sha256": record_sha, + "seed": 0, + "outcome_record": record_out.to_dict() if record_out else None, + } + ) + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / f"{checkpoint_key}__{arm}__{record_id}.json" + path.write_text(json.dumps(cell, indent=2) + "\n", encoding="utf-8") + return cell + + +# --------------------------------------------------------------------------- +# Report (census + sweep cells → disposition + durable docs) +# --------------------------------------------------------------------------- + + +def _predeclaration() -> dict[str, Any]: + return { + "experiment_id": "SLM-303", + "hypothesis": ( + "A material share of historical zero/interference-flagged " + "scoreboards are runtime/harness budget artifacts (decode timeout " + "or fallback), not model behavior; widening the decode budget 10x " + "flips timeout-classified rows to model outcomes." + ), + "falsifier": ( + "10x budget does not change any timeout-classified record's " + "outcome on the hash-pinned checkpoints, or the census shows the " + "flagged scoreboards were already clean model behavior." + ), + "arms": { + "baseline": "recorded canvas cap (256/128) + inferred 10s timeout", + "budget10x": "10x timeout and 10x canvas cap; nothing else varies", + }, + "budget_evidence": BASELINE_TIMEOUT_EVIDENCE, + "records": { + "ids": list(SMOKE_RECORD_IDS), + "source": str(TEST_SEEDS), + "disclosure": ( + "the original eval suite directory is empty; the sweep " + "rebuilds the same 3 smoke records (ids match the recorded " + "scoreboard details) from test_seeds.jsonl" + ), + }, + "seed": 0, + "fixed": ( + "same checkpoint bytes (hash-pinned), same records, same seed, " + "checkpoint-declared decode policy; arms differ only in budget" + ), + "checkpoints": { + key: { + "path": spec["path"], + "recorded_canvas_cap": spec["recorded_canvas_cap"], + "recorded_decode_timeout_count": spec["recorded_decode_timeout_count"], + "selection": "only checkpoints with ACTUAL nonzero-timeout scoreboards", + } + for key, spec in SWEEP_CHECKPOINTS.items() + }, + "excluded": { + "slm230_bounded_recursive_r4_r2": ( + "zero-timeout scoreboard; budget interference not implicated — not needed" + ), + "e173/slm294 and all other local checkpoints": ( + "no committed SHA and no hash-pinned timeout scoreboard → not " + "hash-verifiable for this audit (slm294 additionally " + "feasibility-excluded: >280s/16 tokens constrained decode on rico)" + ), + }, + "claim_class": "diagnostic", + } + + +def _disposition( + census: dict[str, Any], pairs: list[dict[str, Any]] +) -> dict[str, Any]: + timeout_boards = [ + s + for s in census["scoreboards"] + if s["classification"] == "runtime_timeout_interference" + ] + fallback_boards = [ + s + for s in census["scoreboards"] + if s["classification"] == "fallback_interference" + ] + unmeasured_boards = [ + s for s in census["scoreboards"] if s["classification"] == "unmeasured" + ] + flips = [ + p + for p in pairs + if p.get("status") == "paired" + and p.get("outcome_baseline") == "runtime_timeout" + and p.get("outcome_10x") != "runtime_timeout" + ] + not_rerunnable = [p for p in pairs if p.get("outcome_baseline") is None] + annotations = [] + for board in timeout_boards: + annotations.append( + { + "scoreboard": board["scoreboard_path"], + "annotation": ( + f"runtime/harness artifact: {board['decode_timeout_count']}" + f"/{board['n']} decode timeouts at canvas cap " + f"{board['decode_canvas_cap']}; zero parse rate on these " + "rows is budget interference, NOT a model verdict" + ), + } + ) + for board in fallback_boards: + annotations.append( + { + "scoreboard": board["scoreboard_path"], + "annotation": ( + f"fallback interference: fallback_count={board['fallback_count']}; " + "fallback outputs never classify as model success" + ), + } + ) + if not pairs: + sweep_verdict = "sweep cells not run" + elif not_rerunnable: + sweep_verdict = ( + f"not_rerunnable: {len(not_rerunnable)}/{len(pairs)} paired cells " + "blocked by the output-contract gate (checkpoint v0 vs required " + "symbol_only/v2, no migration path); the 10x-budget flip question " + "is UNANSWERED by re-decode — census + taxonomy carry the audit" + ) + else: + sweep_verdict = ( + f"{len(flips)} timeout-classified rows flipped outcome under 10x budget" + ) + recommendation = { + "powered_rerun": ( + "at most one: retrain the remediated recipe from symbol-only " + "targets (output contract v2 — the v0 checkpoints are blocked by " + "require_current_output_contract with no migration path), then a " + "preregistered full-smoke re-eval at recorded vs 10x decode " + "budget with per-record decode_outcome fields (now emitted by " + "eval_runner), n≥16 smoke+fixture records for Wilson resolution" + ), + "rationale": ( + "census localizes all timeout interference to the two remediated " + "checkpoints; no other hash-verifiable checkpoint carries a " + "timeout-flagged scoreboard, and re-decode of the v0 checkpoints " + "under current code is impossible" + ), + } + return { + "annotations": annotations, + "counts": { + "runtime_timeout_interference": len(timeout_boards), + "fallback_interference": len(fallback_boards), + "unmeasured": len(unmeasured_boards), + }, + "sweep_verdict": sweep_verdict, + "recommendation": recommendation, + } + + +def render_markdown(payload: dict[str, Any]) -> str: + summary = payload["census"]["summary"] + lines = [ + "# SLM-303 (LAR1-05): decode-budget audit — census, 10x sweep, disposition", + "", + f"- generated_at: `{payload['generated_at']}`", + f"- claim_class: `{payload['predeclaration']['claim_class']}` (diagnostic; not ship evidence)", + f"- taxonomy: `{', '.join(payload['census']['taxonomy'])}`", + f"- budget evidence: {payload['predeclaration']['budget_evidence']}", + "", + "## Census", + "", + f"- local checkpoints enumerated: {summary['n_local_checkpoints']} " + f"(roster rows: {summary['n_roster_rows']})", + f"- committed SHA verified: {summary['n_committed_sha_verified']}; " + f"mismatch: {summary['n_committed_sha_mismatch']}; " + f"unverifiable (no committed SHA): {summary['n_unverifiable']}", + f"- scoreboard classes: `{json.dumps(summary['scoreboard_classes'])}`", + "", + "### Hash pins (sweep checkpoints)", + "", + "| key | path | sha256 | matches scoreboard record |", + "| --- | --- | --- | --- |", + ] + for key, pin in payload["census"]["hash_pins"].items(): + lines.append( + f"| {key} | `{pin['path']}` | `{pin.get('actual_sha256')}` | " + f"{pin.get('matches_scoreboard_record')} |" + ) + lines += ["", "## Budget sweep (preregistered, paired)", ""] + sweep = payload.get("sweep") or {} + if sweep.get("pairs"): + lines += [ + "| checkpoint | record | baseline outcome | 10x outcome | baseline ms | 10x ms |", + "| --- | --- | --- | --- | --- | --- |", + ] + for pair in sweep["pairs"]: + if pair.get("status") != "paired": + lines.append( + f"| {pair['checkpoint']} | {pair['record_id']} | unpaired | | | |" + ) + continue + lines.append( + f"| {pair['checkpoint']} | {pair['record_id']} | " + f"{pair['outcome_baseline'] or pair.get('cell_status_baseline')} | " + f"{pair['outcome_10x'] or pair.get('cell_status_10x')} | " + f"{pair['elapsed_ms_baseline']} | {pair['elapsed_ms_10x']} |" + ) + lines += [ + "", + f"- outcome counts (baseline): `{json.dumps(sweep['outcome_counts']['baseline'])}`", + f"- outcome counts (10x): `{json.dumps(sweep['outcome_counts']['budget10x'])}`", + ] + else: + lines.append("- sweep cells not present; see census + disposition only") + lines += ["", "## Disposition (append-only annotations)", ""] + disp = payload["disposition"] + for note in disp["annotations"]: + lines.append(f"- `{note['scoreboard']}`: {note['annotation']}") + lines += [ + "", + f"- sweep verdict: {disp['sweep_verdict']}", + f"- powered-rerun recommendation: {disp['recommendation']['powered_rerun']}", + f" - rationale: {disp['recommendation']['rationale']}", + "", + "Historical iter docs and MODEL_CARD rows are untouched; these " + "annotations are additive.", + "", + ] + return "\n".join(lines) + + +def run_report( + *, + json_out: Path = DEFAULT_JSON_OUT, + md_out: Path = DEFAULT_MD_OUT, +) -> dict[str, Any]: + census = run_census() + cells: list[dict[str, Any]] = [] + if CELLS_DIR.exists(): + for path in sorted(CELLS_DIR.glob("*.json")): + cells.append(json.loads(path.read_text(encoding="utf-8"))) + pairs = pair_cells(cells) + outcomes_by_arm = { + arm: outcome_counts( + [c["outcome"] for c in cells if c["arm"] == arm and c["outcome"]] + ) + for arm in ("baseline", "budget10x") + } + payload: dict[str, Any] = { + "schema": "slm303_decode_budget_audit/v1", + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "predeclaration": _predeclaration(), + "census": census, + "sweep": { + "n_cells": len(cells), + "pairs": pairs, + "outcome_counts": outcomes_by_arm, + "cells": cells, + }, + "disposition": _disposition(census, pairs), + } + payload["version_stamp"] = build_version_stamp(COMPONENT, EVAL_COMPONENT) + json_out.write_text( + json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8" + ) + md_out.write_text(render_markdown(payload), encoding="utf-8") + return payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="mode", required=True) + sub.add_parser("census", help="checkpoint/scoreboard census + hash pins") + cell = sub.add_parser("sweep-cell", help="run one (checkpoint, arm, record) cell") + cell.add_argument("--checkpoint", required=True, choices=sorted(SWEEP_CHECKPOINTS)) + cell.add_argument("--arm", required=True, choices=("baseline", "budget10x")) + cell.add_argument("--record-id", required=True, choices=list(SMOKE_RECORD_IDS)) + rep = sub.add_parser("report", help="merge census + cells into durable docs") + rep.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + rep.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + args = parser.parse_args(argv) + + if args.mode == "census": + census = run_census() + print(json.dumps(census["summary"], indent=2)) + print(f"wrote {CENSUS_JSON}") + return 0 + if args.mode == "sweep-cell": + result = run_sweep_cell( + checkpoint_key=args.checkpoint, + arm=args.arm, + record_id=args.record_id, + ) + print( + f"{args.checkpoint}/{args.arm}/{args.record_id}: " + f"{result['outcome']} ({result['status']}, {result['elapsed_ms']}ms)" + ) + return 0 + payload = run_report(json_out=args.json_out, md_out=args.md_out) + print(f"sweep verdict: {payload['disposition']['sweep_verdict']}") + print(f"wrote {args.json_out} and {args.md_out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/model_build/decode_outcome.py b/src/slm_training/harnesses/model_build/decode_outcome.py new file mode 100644 index 000000000..b447aac44 --- /dev/null +++ b/src/slm_training/harnesses/model_build/decode_outcome.py @@ -0,0 +1,140 @@ +"""SLM-303 (LAR1-05): decode-outcome taxonomy for budget/interference audits. + +Every decode attempt lands in exactly one of ``DECODE_OUTCOMES``. The point of +the taxonomy is to stop laundering runtime/harness artifacts (timeouts, +fallbacks, harness exceptions) into model-quality claims: a fallback output is +never a model success, and a timeout says something about the budget, not the +model. + +Classification precedence is strict and total: + + harness_error > runtime_timeout > fallback_output > model_abstain + > model_invalid / model_valid +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any, Mapping + +DECODE_OUTCOMES: tuple[str, ...] = ( + "model_valid", + "model_invalid", + "model_abstain", + "runtime_timeout", + "fallback_output", + "harness_error", +) + +MODEL_VALID, MODEL_INVALID, MODEL_ABSTAIN = DECODE_OUTCOMES[0:3] +RUNTIME_TIMEOUT, FALLBACK_OUTPUT, HARNESS_ERROR = DECODE_OUTCOMES[3:6] + +# DecodeStats counter names that mean "the emitted output came from a +# non-model fallback path". Mirrors the eval_runner fallback_count set. +FALLBACK_COUNTER_NAMES: tuple[str, ...] = ( + "unconstrained_retries", + "compiler_fallbacks", + "seeded_fallbacks", + "template_fallback_count", + "certified_fallbacks", +) + + +def fallback_counter_total(stats: object | None) -> int: + """Total fallback counter value across a DecodeStats-like row (0 if none).""" + if stats is None: + return 0 + if isinstance(stats, Mapping): + return sum(int(stats.get(name) or 0) for name in FALLBACK_COUNTER_NAMES) + return sum(int(getattr(stats, name, 0) or 0) for name in FALLBACK_COUNTER_NAMES) + + +def classify_decode_outcome( + *, + parse_ok: bool | None, + error: str | None = None, + fallback_counters: int = 0, + timed_out: bool = False, + abstained: bool = False, + harness_exception: bool = False, +) -> str: + """Classify one decode attempt into the taxonomy. + + Precedence is strict: a harness exception dominates everything; a timeout + is a budget artifact, not a model verdict; a fallback output is never a + model success even when it happens to parse; an abstention (empty output + without timeout/fallback) outranks a parse verdict. + """ + if harness_exception: + return HARNESS_ERROR + if timed_out: + return RUNTIME_TIMEOUT + if fallback_counters > 0: + return FALLBACK_OUTPUT + if abstained: + return MODEL_ABSTAIN + # parse_ok None = parse not evaluated; a produced candidate with no + # contrary evidence is a model output, not a failure verdict. + return MODEL_INVALID if parse_ok is False else MODEL_VALID + + +@dataclass +class DecodeOutcomeRecordV1: + """One decode attempt with its budget, evidence, and classified outcome.""" + + request_id: str + outcome: str + stop_reason: str + budget: dict[str, Any] = field(default_factory=dict) + elapsed_ms: float | None = None + forwards: int | None = None + verifier_calls: int | None = None + fallback_used: bool = False + detail: str | None = None + + schema: str = "decode_outcome_record/v1" + + def __post_init__(self) -> None: + if self.outcome not in DECODE_OUTCOMES: + raise ValueError(f"unknown decode outcome {self.outcome!r}") + if self.outcome == FALLBACK_OUTPUT and not self.fallback_used: + raise ValueError("fallback_output outcome requires fallback_used=True") + if self.outcome == MODEL_VALID and self.fallback_used: + raise ValueError("a fallback output may never classify as model_valid") + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "DecodeOutcomeRecordV1": + return cls( + request_id=str(data["request_id"]), + outcome=str(data["outcome"]), + stop_reason=str(data["stop_reason"]), + budget=dict(data.get("budget") or {}), + elapsed_ms=data.get("elapsed_ms"), + forwards=data.get("forwards"), + verifier_calls=data.get("verifier_calls"), + fallback_used=bool(data.get("fallback_used", False)), + detail=data.get("detail"), + ) + + +def outcome_counts(outcomes: list[str]) -> dict[str, int]: + """Per-taxonomy counts with every key present (zeros included).""" + counts = {name: 0 for name in DECODE_OUTCOMES} + for outcome in outcomes: + if outcome not in counts: + raise ValueError(f"unknown decode outcome {outcome!r}") + counts[outcome] += 1 + return counts + + +__all__ = [ + "DECODE_OUTCOMES", + "FALLBACK_COUNTER_NAMES", + "DecodeOutcomeRecordV1", + "classify_decode_outcome", + "fallback_counter_total", + "outcome_counts", +] diff --git a/src/slm_training/harnesses/model_build/eval_runner.py b/src/slm_training/harnesses/model_build/eval_runner.py index 6d1524b12..645808776 100644 --- a/src/slm_training/harnesses/model_build/eval_runner.py +++ b/src/slm_training/harnesses/model_build/eval_runner.py @@ -44,6 +44,7 @@ DEFAULT_SHIP_GATES, ) from slm_training.models.decode_stats import collect_decode_stats +from slm_training.harnesses.model_build.decode_outcome import outcome_counts from slm_training.versioning import component_version _COMPONENT_RE = re.compile(r"\b([A-Z][A-Za-z0-9]*)\s*\(") @@ -970,6 +971,11 @@ def _generate_chunk_unbounded( return out, list(evidence) decode_timeout_count = 0 + # SLM-303: per-chunk decode-outcome evidence (parallel to the chunk loop); + # each entry carries the timeout fact plus the chunk's DecodeStats row (or + # None when the plugin exposes no stats) so every scored record can be + # classified into the decode-outcome taxonomy. + chunk_decode_meta: list[dict[str, Any]] = [] def _generate_chunk( chunk: list[ExampleRecord], @@ -977,8 +983,18 @@ def _generate_chunk( """Generate a chunk, converting an explicit diagnostic timeout to failures.""" nonlocal decode_timeout_count seconds = float(getattr(config, "decode_timeout_seconds", 0) or 0) + + def _run(timed_out: bool) -> tuple[list[str], list[dict[str, Any]]]: + stats_before = len(decode_stats_rows) + result = _generate_chunk_unbounded(chunk) + stats = ( + decode_stats_rows[-1] if len(decode_stats_rows) > stats_before else None + ) + chunk_decode_meta.append({"timed_out": timed_out, "stats": stats}) + return result + if seconds <= 0 or not hasattr(signal, "setitimer"): - return _generate_chunk_unbounded(chunk) + return _run(timed_out=False) def _alarm(_signum: int, _frame: object) -> None: raise TimeoutError(f"decode exceeded {seconds:g}s") @@ -986,23 +1002,80 @@ def _alarm(_signum: int, _frame: object) -> None: previous = signal.signal(signal.SIGALRM, _alarm) signal.setitimer(signal.ITIMER_REAL, seconds) try: - return _generate_chunk_unbounded(chunk) + return _run(timed_out=False) except TimeoutError as exc: stats = getattr(exc, "decode_stats", None) if stats is not None: _annotate_decode_trace_records(stats, chunk) decode_stats_rows.append(stats) + chunk_decode_meta.append({"timed_out": True, "stats": stats}) decode_timeout_count += len(chunk) return ["" for _ in chunk], [] finally: signal.setitimer(signal.ITIMER_REAL, 0) signal.signal(signal.SIGALRM, previous) + def _decode_outcome_fields( + pred: str, + *, + parse_ok: bool | None, + error: str | None, + decode_meta: dict[str, Any] | None, + ) -> dict[str, Any]: + """SLM-303 additive per-record decode-outcome classification. + + Computed from the same evidence the runner already has (chunk timeout + fact, chunk DecodeStats fallback counters, parse verdict); never + changes any existing field. + """ + from slm_training.harnesses.model_build.decode_outcome import ( + MODEL_VALID, + classify_decode_outcome, + fallback_counter_total, + ) + + meta = decode_meta or {} + stats = meta.get("stats") + fallbacks = fallback_counter_total(stats) + timed_out = bool(meta.get("timed_out")) + outcome = classify_decode_outcome( + parse_ok=parse_ok, + error=error, + fallback_counters=fallbacks, + timed_out=timed_out, + abstained=not pred.strip(), + harness_exception=False, + ) + if timed_out: + stop_reason = "decode_timeout" + else: + stop_reason = "" + for name in ( + "compiler_lattice_termination_reason", + "solver_terminal_status", + ): + value = str(getattr(stats, name, "") or "") if stats is not None else "" + if value: + stop_reason = value + break + if not stop_reason: + stop_reason = "empty_prediction" if not pred.strip() else "completed" + detail = None + if outcome == MODEL_VALID and parse_ok is None: + detail = "parse_not_evaluated" + return { + "decode_outcome": outcome, + "stop_reason": stop_reason, + "fallback_used": fallbacks > 0, + "decode_outcome_detail": detail, + } + def _score_one( record: ExampleRecord, pred: str, latency_ms: float, prediction_evidence: dict[str, Any] | None = None, + decode_meta: dict[str, Any] | None = None, ) -> None: nonlocal parse_ok, syntax_parse_ok, raw_syntax_ok nonlocal match_error_count, reward_error_count, empty_prediction_count @@ -1036,6 +1109,9 @@ def _score_one( ).encode("utf-8") ).hexdigest(), "topology_evidence": evidence or None, + **_decode_outcome_fields( + pred, parse_ok=None, error=None, decode_meta=decode_meta + ), } ) task_cases.append( @@ -1169,6 +1245,9 @@ def _score_one( ).hexdigest(), "serialized": serialized, "topology_evidence": evidence or None, + **_decode_outcome_fields( + pred, parse_ok=ok, error=error, decode_meta=decode_meta + ), } ) task_cases.append( @@ -1194,16 +1273,18 @@ def _score_one( chunk = records[start : start + batch_size] t0 = time.perf_counter() preds, evidence_rows = _generate_chunk(chunk) + chunk_meta = chunk_decode_meta[-1] if chunk_decode_meta else None elapsed = (time.perf_counter() - t0) * 1000.0 per = elapsed / max(1, len(chunk)) for index, (record, pred) in enumerate(zip(chunk, preds)): latencies.append(per) evidence = evidence_rows[index] if index < len(evidence_rows) else None - _score_one(record, pred, per, evidence) + _score_one(record, pred, per, evidence, chunk_meta) else: for record in records: t0 = time.perf_counter() predictions, evidence_rows = _generate_chunk([record]) + chunk_meta = chunk_decode_meta[-1] if chunk_decode_meta else None pred = predictions[0] latencies.append((time.perf_counter() - t0) * 1000.0) _score_one( @@ -1211,6 +1292,7 @@ def _score_one( pred, latencies[-1], evidence_rows[0] if evidence_rows else None, + chunk_meta, ) lat_sorted = sorted(latencies) @@ -1392,6 +1474,11 @@ def _rate_evidence(successes: int, total: int) -> dict[str, Any]: "failure_breakdown": failure_breakdown, "decode_timeout_count": decode_timeout_count, "decode_canvas_cap": canvas_cap, + # SLM-303: per-taxonomy decode-outcome counts over details[] rows + # (additive; every taxonomy key always present). + "decode_outcome_counts": outcome_counts( + [str(row.get("decode_outcome")) for row in details] + ), "details": details, "generation_evidence_schemas": sorted( { diff --git a/src/slm_training/harnesses/model_build/ship_gates.py b/src/slm_training/harnesses/model_build/ship_gates.py index 9b9673e77..953f8f203 100644 --- a/src/slm_training/harnesses/model_build/ship_gates.py +++ b/src/slm_training/harnesses/model_build/ship_gates.py @@ -66,7 +66,7 @@ # (candidate_pending_calibration) so recording it can never green a gate. MEANINGFUL_METRIC_POLICY = { "active_primary": "meaningful_program_v1", - "threshold_version": "openui_ship_gates_v2", + "threshold_version": "openui_ship_gates_v3", "meaningful_program_v1": { "version": "1.0.0", "wire_field": "meaningful_program_rate", diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 40e8c67b4..45fadec38 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -4521,8 +4521,26 @@ } ] }, + "harness.experiments.slm303_decode_budget_audit": { + "version": "v1", + "kind": "harness", + "paths": [ + "scripts/run_slm303_decode_budget_audit.py", + "tests/test_harnesses/model_build/test_decode_outcome.py", + "tests/test_scripts/test_run_slm303_decode_budget_audit.py", + "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "docs/design/iter-slm303-decode-budget-audit-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-303 decode-budget audit with hash pins, preregistered budget sweep, and taxonomy disposition" + } + ] + }, "harness.model_build.eval": { - "version": "v52", + "version": "v53", "kind": "harness", "paths": [ "src/slm_training/harnesses/model_build/debt_router.py", @@ -4533,6 +4551,11 @@ "scripts/evaluate_model.py" ], "history": [ + { + "version": "v53", + "date": "2026-07-24", + "note": "Add SLM-303 per-record decode outcome taxonomy and suite outcome counts without changing existing gate inputs" + }, { "version": "v52", "date": "2026-07-24", diff --git a/tests/test_harnesses/model_build/test_decode_outcome.py b/tests/test_harnesses/model_build/test_decode_outcome.py new file mode 100644 index 000000000..7020f8001 --- /dev/null +++ b/tests/test_harnesses/model_build/test_decode_outcome.py @@ -0,0 +1,115 @@ +"""SLM-303: decode-outcome taxonomy tests.""" + +from __future__ import annotations + +import pytest + +from slm_training.harnesses.model_build.decode_outcome import ( + DECODE_OUTCOMES, + DecodeOutcomeRecordV1, + classify_decode_outcome, + fallback_counter_total, + outcome_counts, +) + + +def test_taxonomy_has_six_outcomes() -> None: + assert DECODE_OUTCOMES == ( + "model_valid", + "model_invalid", + "model_abstain", + "runtime_timeout", + "fallback_output", + "harness_error", + ) + + +def test_precedence_harness_error_dominates() -> None: + assert ( + classify_decode_outcome( + parse_ok=True, + fallback_counters=3, + timed_out=True, + abstained=True, + harness_exception=True, + ) + == "harness_error" + ) + + +def test_precedence_timeout_beats_fallback_and_parse() -> None: + assert ( + classify_decode_outcome( + parse_ok=True, fallback_counters=2, timed_out=True + ) + == "runtime_timeout" + ) + + +def test_fallback_never_model_success() -> None: + # Even a parseable output is a fallback artifact when a fallback fired. + assert ( + classify_decode_outcome(parse_ok=True, fallback_counters=1) + == "fallback_output" + ) + assert ( + classify_decode_outcome(parse_ok=False, fallback_counters=1) + == "fallback_output" + ) + + +def test_abstain_beats_parse_verdict() -> None: + assert classify_decode_outcome(parse_ok=False, abstained=True) == "model_abstain" + + +def test_model_valid_and_invalid() -> None: + assert classify_decode_outcome(parse_ok=True) == "model_valid" + assert classify_decode_outcome(parse_ok=False) == "model_invalid" + # parse not evaluated + produced output → model_valid with no parse claim + assert classify_decode_outcome(parse_ok=None) == "model_valid" + + +def test_fallback_counter_total_mapping_and_object() -> None: + assert fallback_counter_total(None) == 0 + assert fallback_counter_total({"compiler_fallbacks": 2, "seeded_fallbacks": 1}) == 3 + + class _Stats: + unconstrained_retries = 1 + template_fallback_count = 2 + + assert fallback_counter_total(_Stats()) == 3 + + +def test_outcome_counts_all_keys_present() -> None: + counts = outcome_counts(["model_valid", "model_valid", "runtime_timeout"]) + assert counts["model_valid"] == 2 + assert counts["runtime_timeout"] == 1 + assert counts["harness_error"] == 0 + assert set(counts) == set(DECODE_OUTCOMES) + with pytest.raises(ValueError): + outcome_counts(["not_an_outcome"]) + + +def test_record_roundtrip_and_guards() -> None: + record = DecodeOutcomeRecordV1( + request_id="r1", + outcome="model_valid", + stop_reason="completed", + budget={"timeout_s": 10.0, "canvas_cap": 256}, + elapsed_ms=12.5, + ) + clone = DecodeOutcomeRecordV1.from_dict(record.to_dict()) + assert clone == record + with pytest.raises(ValueError): + DecodeOutcomeRecordV1(request_id="r", outcome="bogus", stop_reason="x") + with pytest.raises(ValueError): + DecodeOutcomeRecordV1( + request_id="r", outcome="fallback_output", stop_reason="x" + ) + with pytest.raises(ValueError): + DecodeOutcomeRecordV1( + request_id="r", + outcome="model_valid", + stop_reason="x", + fallback_used=True, + ) diff --git a/tests/test_scripts/test_run_slm303_decode_budget_audit.py b/tests/test_scripts/test_run_slm303_decode_budget_audit.py new file mode 100644 index 000000000..56b337d53 --- /dev/null +++ b/tests/test_scripts/test_run_slm303_decode_budget_audit.py @@ -0,0 +1,159 @@ +"""SLM-303: census / hash-pin / sweep-pairing tests (pure functions only).""" + +from __future__ import annotations + +import pytest + +from scripts.run_slm303_decode_budget_audit import ( + SWEEP_CHECKPOINTS, + build_sweep_arms, + classify_scoreboard, + pair_cells, + sha_matches_committed, +) + +ACTUAL = "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a" + + +def test_sha_fragment_prefix_suffix_match() -> None: + assert sha_matches_committed("653e71b2…f8fa3a", ACTUAL) is True + assert sha_matches_committed("`653e71b2…f8fa3a`", ACTUAL) is True + assert sha_matches_committed("deadbeef…f8fa3a", ACTUAL) is False + assert sha_matches_committed("653e71b2…0000", ACTUAL) is False + + +def test_sha_full_match_and_mismatch() -> None: + assert sha_matches_committed(ACTUAL, ACTUAL) is True + assert sha_matches_committed("0" * 64, ACTUAL) is False + + +def test_sha_uncommitted_is_unverifiable_not_pass() -> None: + assert sha_matches_committed(None, ACTUAL) is None + assert sha_matches_committed("", ACTUAL) is None + assert sha_matches_committed("not-a-sha", ACTUAL) is None + + +def _scoreboard(**overrides): + base = { + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "details": [], + } + base.update(overrides) + return base + + +def test_classify_scoreboard_unmeasured_when_fallback_null() -> None: + result = classify_scoreboard(_scoreboard(fallback_count=None)) + assert result["classification"] == "unmeasured" + + +def test_classify_scoreboard_timeout_interference() -> None: + result = classify_scoreboard(_scoreboard(decode_timeout_count=3)) + assert result["classification"] == "runtime_timeout_interference" + + +def test_classify_scoreboard_fallback_interference() -> None: + result = classify_scoreboard(_scoreboard(fallback_count=2)) + assert result["classification"] == "fallback_interference" + + +def test_classify_scoreboard_clean_zero_is_model_behavior() -> None: + result = classify_scoreboard(_scoreboard()) + assert result["classification"] == "model_behavior" + + +def test_classify_scoreboard_missing_fields_unmeasured() -> None: + assert classify_scoreboard({"n": 3})["classification"] == "unmeasured" + + +def test_classify_scoreboard_row_outcomes_from_details() -> None: + board = classify_scoreboard( + _scoreboard( + details=[ + {"id": "a", "parse_ok": True, "prediction": "(root)"}, + {"id": "b", "parse_ok": False, "prediction": "junk"}, + {"id": "c", "parse_ok": False, "prediction": ""}, + ] + ) + ) + outcomes = {r["id"]: r["outcome"] for r in board["row_outcomes"]} + assert outcomes == { + "a": "model_valid", + "b": "model_invalid", + "c": "model_abstain", + } + + +def test_sweep_arms_differ_only_in_budget() -> None: + arms = build_sweep_arms(256) + assert set(arms) == {"baseline", "budget10x"} + assert arms["baseline"]["canvas_cap"] == 256 + assert arms["budget10x"]["canvas_cap"] == 2560 + assert arms["budget10x"]["timeout_s"] == 10 * arms["baseline"]["timeout_s"] + + +def _cell(checkpoint: str, arm: str, record: str, outcome: str, sha: str = "abc"): + return { + "checkpoint": checkpoint, + "arm": arm, + "record_id": record, + "outcome": outcome, + "budget": {"timeout_s": 10.0, "canvas_cap": 256}, + "checkpoint_sha256": sha, + "record_sha256": "rec", + "seed": 0, + "elapsed_ms": 5.0, + "parse_ok": None, + } + + +def test_pair_cells_pairs_and_detects_flips() -> None: + cells = [ + _cell("ltr2", "baseline", "smoke_hero_01", "runtime_timeout"), + _cell("ltr2", "budget10x", "smoke_hero_01", "model_invalid"), + ] + (pair,) = pair_cells(cells) + assert pair["status"] == "paired" + assert pair["outcome_baseline"] == "runtime_timeout" + assert pair["outcome_10x"] == "model_invalid" + + +def test_pair_cells_unpaired_and_violation() -> None: + unpaired = pair_cells([_cell("ltr2", "baseline", "smoke_hero_01", "x")]) + assert unpaired[0]["status"] == "unpaired" + violation = pair_cells( + [ + _cell("ltr2", "baseline", "smoke_hero_01", "a", sha="abc"), + _cell("ltr2", "budget10x", "smoke_hero_01", "b", sha="def"), + ] + ) + assert violation[0]["status"] == "pair_violation" + assert "checkpoint_sha256" in violation[0]["pair_violation_fields"] + + +def test_pairing_is_deterministic() -> None: + cells = [ + _cell("ltr2", "budget10x", "smoke_button_01", "model_invalid"), + _cell("ltr2", "baseline", "smoke_hero_01", "runtime_timeout"), + _cell("ltr2", "baseline", "smoke_button_01", "runtime_timeout"), + _cell("ltr2", "budget10x", "smoke_hero_01", "model_invalid"), + ] + assert pair_cells(cells) == pair_cells(list(reversed(cells))) + + +def test_sweep_checkpoints_are_the_two_remediated() -> None: + assert set(SWEEP_CHECKPOINTS) == {"ltr2", "lexer_ltr2"} + for spec in SWEEP_CHECKPOINTS.values(): + assert spec["recorded_decode_timeout_count"] == 3 + assert spec["recorded_canvas_cap"] in (128, 256) + + +@pytest.mark.parametrize("cap", [128, 256]) +def test_arms_track_recorded_cap(cap: int) -> None: + arms = build_sweep_arms(cap) + assert arms["baseline"]["canvas_cap"] == cap + assert arms["budget10x"]["canvas_cap"] == cap * 10 From dbe2fd595f6a40a964e652ebb1a7bca3df6e4d97 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:29:17 -0500 Subject: [PATCH 03/10] SLM-305: extend tree edit language --- README.md | 12 + docs/design/agentv-evaluation.md | 5 + .../iter-slm305-edit-language-20260724.json | 732 ++++++++++++++++++ .../iter-slm305-edit-language-20260724.md | 102 +++ .../langsmith-telemetry-smoke-20260724.json | 51 ++ package-lock.json | 180 ++++- package.json | 4 + pyproject.toml | 4 + scripts/run_agentv_eval.mjs | 55 ++ scripts/run_slm299_reachability_audit.py | 121 ++- src/slm_training/evals/agentv.py | 22 +- .../experiments/slm299_edit_reachability.py | 273 ++++++- .../harnesses/model_build/eval_runner.py | 58 ++ src/slm_training/models/checkpoint_migrate.py | 94 +++ .../models/tree_edit_diffusion.py | 613 ++++++++++++++- src/slm_training/resources/versions.json | 24 +- src/slm_training/runtime/telemetry/trace.py | 199 ++++- tests/test_evals/test_agentv.py | 34 + .../test_slm299_edit_reachability.py | 108 ++- .../model_build/test_eval_gates.py | 39 + tests/test_models/test_tree_edit_diffusion.py | 160 ++++ tests/test_runtime_trace.py | 145 ++++ 22 files changed, 2915 insertions(+), 120 deletions(-) create mode 100644 docs/design/iter-slm305-edit-language-20260724.json create mode 100644 docs/design/iter-slm305-edit-language-20260724.md create mode 100644 docs/design/langsmith-telemetry-smoke-20260724.json diff --git a/README.md b/README.md index 4b01aa277..f0c4cfeeb 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,18 @@ instead of memorizing paths. Selected immutable snapshots publish to Git with Every new run writes `outputs/runs//trace.json` and OTLP JSONL signals under `outputs/traces//`. Set `OTEL_EXPORTER_OTLP_ENDPOINT` for an optional remote OTLP mirror; detailed domain traces remain local and linked by trace ID. +Set `LANGSMITH_TRACING=true` and provide `LANGSMITH_API_KEY` to export an +additional, best-effort aggregate trace to the `slm-training` LangSmith project +(`LANGSMITH_PROJECT`, `LANGSMITH_ENDPOINT`, and `LANGSMITH_WORKSPACE_ID` are +optional overrides). This exports run/suite metrics, version stamps, gate +verdicts, and AgentV summaries only—never prompts, targets, completions, +checkpoints, raw logs, or credentials. Local evidence and AgentEvals remain +authoritative when LangSmith is unavailable. Install the harness-side client +with `pip install -e '.[observability]'`; the shared trace boundary loads the +repository's ignored `.env` without overriding environment variables provided +by CI or a shell. Standard OTLP settings are honored as well: +`OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`, signal-specific endpoints, +and `OTEL_EXPORTER_OTLP_TIMEOUT` (milliseconds; bounded to five seconds). The flush pipeline remains: curated seeds + RICO + Awwwards → deterministic quality synth → per-record DESIGN.md + OpenUI validate → quality gates → stable diff --git a/docs/design/agentv-evaluation.md b/docs/design/agentv-evaluation.md index 1aa131839..12f7a6450 100644 --- a/docs/design/agentv-evaluation.md +++ b/docs/design/agentv-evaluation.md @@ -29,6 +29,10 @@ AgentEvals assertion results has `authority: "AgentEvals assertions"`. under `/evals/`. 5. `write_ship_gates` projects those assertion results into the compatibility `gates.json` shape and records the AgentEvals authority. +6. When opt-in LangSmith tracing is configured, the runner publishes its + aggregate AgentV summary as a child of the existing W3C-correlated run + trace. This is observational only; AgentEvals assertions, local artifacts, + and ship gates retain authority if export fails. The `agentv` npm package is retained for the canonical CLI and dashboard. In the pinned release its published package is CLI-only, so programmatic execution @@ -82,3 +86,4 @@ The implementation check is recorded in | --- | --- | --- | --- | | 2026-07-14 | CPU, steps 0, no model backend; AgentV SDK fixture plus model/loss/task/train-loop harness tests | 34/34 focused tests passed; SDK fixture wrote valid AgentEvals JSONL and AgentV artifacts; dependency audit has 0 high/critical findings | Tooling wiring only; no checkpoint, model score, or ship gate was produced | | 2026-07-23 | CPU, steps 0, no model backend; focused assertion-authority tests with the pinned SDK | AgentEvals JSONL carried required code-graders over raw criteria; the runner produced 1/1 passing fixture criteria with 0 execution errors; 135 focused gate/consumer checks and all 6 interpreted page validations passed. Dashboard production build remained environment-blocked because the locked `@openfeature/web-sdk` dependency was unavailable and automatic review rejected installation before execution. | Tooling and gate-authority wiring only; no checkpoint or model-quality claim | +| 2026-07-24 | Local summary-only telemetry configuration smoke; LangSmith `slm-training` project, two-second bounded flush, standard OTLP resource defaults | W3C root trace `fdf78e5360a6647b09fa0882a2c5d4ec` was published and read back as `slm.telemetry.config` | Tooling wiring only; no checkpoint, model score, or ship gate was produced ([JSON](langsmith-telemetry-smoke-20260724.json)) | diff --git a/docs/design/iter-slm305-edit-language-20260724.json b/docs/design/iter-slm305-edit-language-20260724.json new file mode 100644 index 000000000..6d83d4525 --- /dev/null +++ b/docs/design/iter-slm305-edit-language-20260724.json @@ -0,0 +1,732 @@ +{ + "schema": "slm299_edit_reachability_audit/v2", + "experiment_id": "slm299-edit-reachability", + "seed_source": "root = Stack([], \"column\")", + "mode": "extended", + "max_edits": 8, + "node_budget": 120, + "generated_at": "2026-07-24T20:22:57Z", + "verdict_policy": "reachable_fraction is computed over decided cases only; UNKNOWN_BUDGET cases are reported separately and are never counted as unreachable; suites without a corpus are corpus_unavailable, never zero-reachable. Reachability is a space-coverage proof, never a model-quality claim.", + "suites": { + "train": { + "status": "ok", + "n_cases": 6, + "n_decided": 6, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_direction_change": 3, + "unsupported_component": 3 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "train_auth_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_auth_01_aug_dir", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_auth_01_syn_2", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01_aug_dir", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01_syn_0", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "smoke": { + "status": "ok", + "n_cases": 3, + "n_decided": 3, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 3 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "smoke_hero_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_button_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_callout_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "held_out": { + "status": "ok", + "n_cases": 5, + "n_decided": 5, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 5 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "held_out_form_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_input_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_tabs_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_settings_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "adversarial": { + "status": "ok", + "n_cases": 4, + "n_decided": 4, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_direction_change": 3, + "unsupported_component": 1 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "adv_empty_prompt_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_deep_nest_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_many_buttons_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "ood": { + "status": "ok", + "n_cases": 4, + "n_decided": 4, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 4 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "ood_dashboard_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_gallery_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_modal_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_auth_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "rico": { + "status": "ok", + "n_cases": 6, + "n_decided": 6, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_direction_change": 6 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "rico_eval_test_0", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_1", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_2", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_4", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_8", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_9", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + } + }, + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "5d35b90d7fc58c094604c23d702ab65593f98004", + "code_dirty": true, + "components": { + "harness.experiments.slm299_edit_reachability": "v2" + }, + "stamped_at": "2026-07-24T20:22:57.129778+00:00" + }, + "suites_v1": { + "train": { + "status": "ok", + "n_cases": 6, + "n_decided": 6, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_direction_change": 3, + "unsupported_component": 3 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "train_auth_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_auth_01_aug_dir", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_auth_01_syn_2", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01_aug_dir", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01_syn_0", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "smoke": { + "status": "ok", + "n_cases": 3, + "n_decided": 3, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_container_add": 2, + "unsupported_component": 1 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "smoke_hero_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_button_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_callout_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "held_out": { + "status": "ok", + "n_cases": 5, + "n_decided": 5, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_container_add": 3, + "unsupported_component": 2 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "held_out_form_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_input_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_tabs_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_settings_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "adversarial": { + "status": "ok", + "n_cases": 4, + "n_decided": 4, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_container_add": 2, + "needs_direction_change": 2 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "adv_empty_prompt_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_deep_nest_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_many_buttons_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "ood": { + "status": "ok", + "n_cases": 4, + "n_decided": 4, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_container_add": 2, + "unsupported_component": 2 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "ood_dashboard_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_gallery_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_modal_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_auth_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "rico": { + "status": "ok", + "n_cases": 6, + "n_decided": 6, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_container_add": 6 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "rico_eval_test_0", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_1", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_2", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_4", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_8", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_9", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + } + ] + } + }, + "old_vs_extended": { + "train": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "smoke": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "held_out": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "adversarial": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "ood": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "rico": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + } + }, + "x22_evidence_annotations": [ + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "adversarial", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided adversarial cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "adversarial", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided adversarial cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "held_out", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "held_out", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "ood", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "ood", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "rico", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided rico cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "rico", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided rico cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "smoke", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "smoke", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "train", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided train cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on train in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "train", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided train cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on train in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + } + ] +} diff --git a/docs/design/iter-slm305-edit-language-20260724.md b/docs/design/iter-slm305-edit-language-20260724.md new file mode 100644 index 000000000..d9f85ae0f --- /dev/null +++ b/docs/design/iter-slm305-edit-language-20260724.md @@ -0,0 +1,102 @@ +# SLM-299 (LAR1-03): X22 edit-space reachability audit + +- generated_at: `2026-07-24T20:22:57Z` +- seed: `root = Stack([], "column")` +- mode: `extended` +- max_edits: 8, node_budget: 120 +- verdict policy: reachable_fraction is computed over decided cases only; UNKNOWN_BUDGET cases are reported separately and are never counted as unreachable; suites without a corpus are corpus_unavailable, never zero-reachable. Reachability is a space-coverage proof, never a model-quality claim. + +> Reachability is space coverage, not model quality: no quality claim +> follows from these proofs alone. + +## Suite summary + +| suite | n | decided | unknown | reachable_fraction | min/med/max edits | +| --- | --- | --- | --- | --- | --- | +| train | 6 | 6 | 0 | 0.0 | — | +| smoke | 3 | 3 | 0 | 0.0 | — | +| held_out | 5 | 5 | 0 | 0.0 | — | +| adversarial | 4 | 4 | 0 | 0.0 | — | +| ood | 4 | 4 | 0 | 0.0 | — | +| rico | 6 | 6 | 0 | 0.0 | — | + +## Reason-code histograms + +- **train**: `{"needs_direction_change": 3, "unsupported_component": 3}` +- **smoke**: `{"unsupported_component": 3}` +- **held_out**: `{"unsupported_component": 5}` +- **adversarial**: `{"needs_direction_change": 3, "unsupported_component": 1}` +- **ood**: `{"unsupported_component": 4}` +- **rico**: `{"needs_direction_change": 6}` + +## Action / component coverage (reachable paths) + +- **train**: actions `{}` components `{}` +- **smoke**: actions `{}` components `{}` +- **held_out**: actions `{}` components `{}` +- **adversarial**: actions `{}` components `{}` +- **ood**: actions `{}` components `{}` +- **rico**: actions `{}` components `{}` + +## Per-case verdicts + +- **train**: + - `train_auth_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `train_auth_01_aug_dir` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `train_auth_01_syn_2` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `train_button_row_01` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `train_button_row_01_aug_dir` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `train_button_row_01_syn_0` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) +- **smoke**: + - `smoke_hero_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `smoke_button_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `smoke_callout_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) +- **held_out**: + - `held_out_form_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_dual_card_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_input_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_tabs_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_settings_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) +- **adversarial**: + - `adv_empty_prompt_01` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `adv_dual_card_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `adv_deep_nest_01` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `adv_many_buttons_01` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) +- **ood**: + - `ood_dashboard_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `ood_gallery_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `ood_modal_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `ood_auth_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) +- **rico**: + - `rico_eval_test_0` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_1` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_2` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_4` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_8` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_9` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + +## X22 evidence annotations (append-only) + +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [adversarial]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided adversarial cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [adversarial]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided adversarial cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [held_out]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [held_out]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [ood]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [ood]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [rico]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided rico cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [rico]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided rico cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [smoke]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [smoke]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [train]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided train cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on train in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [train]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided train cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on train in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. + +## Old (v1) vs extended (SLM-305) reachability + +| suite | reachable v1 | reachable extended | verdict flips | action-cost delta min/med/max | +| --- | --- | --- | --- | --- | +| train | 0.0 | 0.0 | 0 | — | +| smoke | 0.0 | 0.0 | 0 | — | +| held_out | 0.0 | 0.0 | 0 | — | +| adversarial | 0.0 | 0.0 | 0 | — | +| ood | 0.0 | 0.0 | 0 | — | +| rico | 0.0 | 0.0 | 0 | — | diff --git a/docs/design/langsmith-telemetry-smoke-20260724.json b/docs/design/langsmith-telemetry-smoke-20260724.json new file mode 100644 index 000000000..9a0a5ecb3 --- /dev/null +++ b/docs/design/langsmith-telemetry-smoke-20260724.json @@ -0,0 +1,51 @@ +{ + "schema": "langsmith_telemetry_wiring/v1", + "run": { + "kind": "langsmith_telemetry_config_smoke", + "date": "2026-07-24", + "device": "local", + "steps": 0, + "context_backend": "none", + "matrix_set": "telemetry-config", + "suite_n": 0, + "honesty": "tooling_wiring_not_model_eval", + "checkpoint": null, + "ship_gates": { + "status": "not_run", + "pass": null, + "reason": "No model checkpoint or quality suite was evaluated." + } + }, + "configuration": { + "langsmith": { + "enabled": true, + "project": "slm-training", + "api_key": "configured_not_recorded", + "flush_seconds": 2.0, + "payload": "summary_only" + }, + "otlp": { + "service_name": "slm-training", + "resource_namespace": "openui", + "export_timeout_ms": 1000, + "endpoint": "not_configured" + } + }, + "verification": { + "trace_id": "fdf78e5360a6647b09fa0882a2c5d4ec", + "root_run_name": "slm.telemetry.config", + "remote_readback": true, + "remote_run_id_matches_w3c_trace_id": true + }, + "verdict": "configured_and_readback_verified", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "cc2521119d6d89d4663131a7001ce45897ca8cdd", + "code_dirty": true, + "components": { + "harness.model_build.eval": "v52", + "evals.agentv": "v3" + }, + "stamped_at": "2026-07-24T19:21:28.616031+00:00" + } +} diff --git a/package-lock.json b/package-lock.json index c31389ed7..c16585b33 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,10 @@ "": { "name": "slm-training-web", "version": "0.1.0", + "dependencies": { + "langsmith": "^0.8.7", + "openai": "^6.49.0" + }, "devDependencies": { "@agentv/core": "4.42.4", "@playwright/mcp": "0.0.78", @@ -274,7 +278,7 @@ "version": "3.975.2", "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.2.tgz", "integrity": "sha512-iyeXwziyjJpixq5OmhsIyrSWx8vwcI7gDo4yRUC3EP7NQtOo9iAJiIEc3G+/HkhtNXqOhofiCK7Lc34Sq+fJWg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.974.1", @@ -294,7 +298,7 @@ "version": "3.972.58", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.58.tgz", "integrity": "sha512-vyGtvK1rY940eq7JT0yIGKuZ+2kpPSJcHibSvGlit5oiMFDamzC7cxBGLl4FLnd6suihMXDI2FSF2dL6TmBqPA==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -311,7 +315,7 @@ "version": "3.972.60", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.60.tgz", "integrity": "sha512-g9b9YzDrD5pcKiPBJfCSXRfFMrA39eR0guUhZ5SRm+7vMAVc43+effxbcamxBjSd5bUhrdKo5te/yQuWurLXLA==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -330,7 +334,7 @@ "version": "3.973.2", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.2.tgz", "integrity": "sha512-Yr7yxNyQ8aHt9Ww0RPFUZx+xiem+vl7vuwhP0tniTijoesJNV5jou9HCgVpI0GEPAF+89TkOvilE5uRrZJnjaw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -355,7 +359,7 @@ "version": "3.972.64", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.64.tgz", "integrity": "sha512-YQoSI4d6kXvoenoG/0Jv/PqaAuukHzGmGXGyHBQYeEUNsYovlNAn/Sw1wp/WQbhcQ3HsEMGgjEahvD3igz6ecQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -373,7 +377,7 @@ "version": "3.972.68", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.68.tgz", "integrity": "sha512-4akjzW9CjorByYfqXBXmYUh/h7Io3U4DtVgGGh9TQraZ7ZlyJqNyHwDRGiUFnHD+BTOeTbCesCa4sJaK7BGZ7A==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.58", @@ -396,7 +400,7 @@ "version": "3.972.58", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.58.tgz", "integrity": "sha512-1nYitRCaDmXWUrpBJt6WlcGjLx1JVsMY8rlYuHHsTYTSaYikbixYdQSyINN2VYq1F798uTO9qHAzytL25M8g3A==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -413,7 +417,7 @@ "version": "3.973.2", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.2.tgz", "integrity": "sha512-pjMLaLU/JZi5lVfmR14V1OZqRBTuMHf6AwGNZA0K9hK+JKtO3jcLBarfD8iq5oc8cSowvc/9R32sqMVXZPo6xQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -432,7 +436,7 @@ "version": "3.972.64", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.64.tgz", "integrity": "sha512-7Buc7p0OvDHW7iBsu4b+YdS0WnaFBDGKDfbVQqaac9dkWiSiUtIoarBDsA1RmOVXZijaZJDoHJFIQiicQvWRlQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -501,7 +505,7 @@ "version": "3.997.32", "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.32.tgz", "integrity": "sha512-6Yj2fr9XF67cndITea48rchTdVr3VGx6PN47bIKNinJAjLkmaIlz/4EBPCgJ8UmhVopiXmeAuPLI3+DXDDbMhQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -521,7 +525,7 @@ "version": "3.996.40", "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.40.tgz", "integrity": "sha512-wrGZ/authosokclY1DXsiWT/1WjfCI22FuZGgdcilF+XLTXs5dCjAtiFYSPsEToZkbm3Lj2YP8PoWg0yoMNu0g==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.974.1", @@ -537,7 +541,7 @@ "version": "3.1087.0", "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1087.0.tgz", "integrity": "sha512-umM+qNq16f2fH+VLM5MqXW4ORNQAjk+TOSto73xbUHcKaU41L48j786r3UWQYlejeJk37NlvRYgxBT+MBkfaYQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -555,7 +559,7 @@ "version": "3.974.1", "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.1.tgz", "integrity": "sha512-W0IQZR0eaBqlBFIIofMapaWkw1W0U+Xi4dvW+BqwmCEMd8Ng2U6IhkxuPSjMVnR8klLjfuS9PeZWUl1N6UaZdg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.16.1", @@ -569,7 +573,7 @@ "version": "3.972.35", "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.35.tgz", "integrity": "sha512-pXzaWe3evZhjxDXAlMnqISe/XefTCGwBJG4nFTXaWSgAnMkqPEhxEPqJNhhpGesEvKFhvNpnozJJ4GTL11bRYw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.16.1", @@ -583,7 +587,7 @@ "version": "0.3.0", "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=18.0.0" @@ -644,6 +648,28 @@ } } }, + "node_modules/@earendil-works/pi-ai/node_modules/openai": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.26.0.tgz", + "integrity": "sha512-zd23dbWTjiJ6sSAX6s0HrCZi41JwTA1bQVs0wLQPZ2/5o2gxOJA5wh7yOAUgwYybfhDXyhwlpeQf7Mlgx8EOCA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/@github/copilot": { "version": "1.0.70", "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.70.tgz", @@ -1458,7 +1484,6 @@ "version": "1.9.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", - "dev": true, "license": "Apache-2.0", "optional": true, "engines": { @@ -1821,7 +1846,6 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.5.1.tgz", "integrity": "sha512-iZH3Gw8cxQn0gjpOjJMmKLd9GIaNh/E3v3ST67vyzLSxHBs14HsG4dy7jMYyC5WXGdBVEcM7U/XTF5hCQxjDMw==", - "dev": true, "license": "Apache-2.0", "optional": true, "dependencies": { @@ -1840,7 +1864,6 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", - "dev": true, "license": "Apache-2.0", "optional": true, "dependencies": { @@ -1857,7 +1880,6 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.5.1.tgz", "integrity": "sha512-BViBCdE/GuXRlp9k7nS1w6wJvY5fnFX5XvuEtWsTAOQFIO89Eru7lGW3WbfbxtCuZ/GbrJfAziXG0w0dpxL7eQ==", - "dev": true, "license": "Apache-2.0", "optional": true, "dependencies": { @@ -1914,7 +1936,7 @@ "version": "1.43.0", "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=14" @@ -2055,7 +2077,7 @@ "version": "3.29.4", "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.4.tgz", "integrity": "sha512-G1GRglAabzEhqghJMBAd54FkRS7SAFGHEwbhcI9r+O+LIMuFsLyXkLZkCoFSgAglRu8s/URVXJB0hglq3ZipIg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.16.1", @@ -2069,7 +2091,7 @@ "version": "4.4.9", "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.9.tgz", "integrity": "sha512-2nfV4qRKiYeXU4zD2vvSCfg5dfp/BuhrM73vt7q9gzBhxs4rbPxXY21wo+kyI3bRmXcEGRnCLTaW8O437jzHIg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/core": "^3.29.4", @@ -2084,7 +2106,7 @@ "version": "5.6.6", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.6.tgz", "integrity": "sha512-NHLgAlORUFZjn5ZfhYuyyKMlXA1WLYOdGxEhyNxrPpbJzoacGbl0chn1lN2KiZ8mpNVk0tV5607CSYlYs/OFgw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/core": "^3.29.4", @@ -2099,7 +2121,7 @@ "version": "4.9.6", "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.6.tgz", "integrity": "sha512-odd+HYx3OLcXRSEz0ZeF3JQdSYdK8QnRgA2N87cPW7coWIbKfRk7a9VQjfeWQLqnzrDLk23KMEn46p8N7M/JFg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/core": "^3.29.4", @@ -2114,7 +2136,7 @@ "version": "5.6.5", "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.5.tgz", "integrity": "sha512-MO5VEhwVl0BN7xVoVeNrZfiUFoQtqxUbgl6/RwOTlMMxCSjblG8twSrVTwz3J4w9WZxd2rBfBAUXjH77agspBg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/core": "^3.29.4", @@ -2129,7 +2151,7 @@ "version": "4.16.1", "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -2334,7 +2356,7 @@ "version": "2.14.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/braces": { @@ -2678,6 +2700,12 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -3349,6 +3377,39 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/langsmith": { + "version": "0.8.7", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.8.7.tgz", + "integrity": "sha512-QWcc7JwmGy+sPRJwqCf9xLiiSIhwc/i7oVwjQezkx6UzyFlokME3Wxy80qKqirTyJvWG1S9ylqtxBDpFRpas5g==", + "license": "MIT", + "dependencies": { + "p-queue": "6.6.2" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } + } + }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -3554,19 +3615,27 @@ } }, "node_modules/openai": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-6.26.0.tgz", - "integrity": "sha512-zd23dbWTjiJ6sSAX6s0HrCZi41JwTA1bQVs0wLQPZ2/5o2gxOJA5wh7yOAUgwYybfhDXyhwlpeQf7Mlgx8EOCA==", - "dev": true, + "version": "6.49.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.49.0.tgz", + "integrity": "sha512-aYCc0C6L864eR6WSYIwQGyXriw/nIyZx0ObvhzOEVuk0zoBDpynjSbrionWI7q65B5H8jJX0DXR9snEzM6bfPg==", "license": "Apache-2.0", - "bin": { - "openai": "bin/cli" - }, "peerDependencies": { + "@aws-sdk/credential-provider-node": ">=3.972.0 <4", + "@smithy/hash-node": ">=4.3.0 <5", + "@smithy/signature-v4": ">=5.4.0 <6", "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "peerDependenciesMeta": { + "@aws-sdk/credential-provider-node": { + "optional": true + }, + "@smithy/hash-node": { + "optional": true + }, + "@smithy/signature-v4": { + "optional": true + }, "ws": { "optional": true }, @@ -3575,6 +3644,31 @@ } } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-retry": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", @@ -3589,6 +3683,18 @@ "node": ">=8" } }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -4127,7 +4233,7 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, + "devOptional": true, "license": "0BSD" }, "node_modules/type-is": { @@ -4244,7 +4350,7 @@ "version": "8.21.1", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=10.0.0" @@ -4282,7 +4388,7 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", - "dev": true, + "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/package.json b/package.json index 05e70d356..1d4c0d837 100644 --- a/package.json +++ b/package.json @@ -27,5 +27,9 @@ }, "engines": { "node": ">=20 <23" + }, + "dependencies": { + "langsmith": "^0.8.7", + "openai": "^6.49.0" } } diff --git a/pyproject.toml b/pyproject.toml index 36eb95517..3fd1067ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,10 @@ research = [ "openai>=2.0,<3", "trackio>=0.1,<1", ] +observability = [ + "langsmith>=0.9,<1", + "python-dotenv>=1.0,<2", +] web = [ "fastapi>=0.110,<1", "onnx>=1.16,<2", diff --git a/scripts/run_agentv_eval.mjs b/scripts/run_agentv_eval.mjs index 7d24a7ba1..5a0d73657 100644 --- a/scripts/run_agentv_eval.mjs +++ b/scripts/run_agentv_eval.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node import { pathToFileURL } from "node:url"; +import { randomUUID } from "node:crypto"; function option(name) { const index = process.argv.indexOf(name); @@ -10,10 +11,62 @@ function option(name) { return process.argv[index + 1]; } +function optional(name) { + const index = process.argv.indexOf(name); + return index === -1 ? undefined : process.argv[index + 1]; +} + +function langsmithEnabled() { + return /^(1|true|yes|on)$/i.test(process.env.LANGSMITH_TRACING ?? "") + && Boolean(process.env.LANGSMITH_API_KEY); +} + +function traceUuid(traceId) { + return traceId && /^[0-9a-f]{32}$/i.test(traceId) + ? `${traceId.slice(0, 8)}-${traceId.slice(8, 12)}-${traceId.slice(12, 16)}-${traceId.slice(16, 20)}-${traceId.slice(20)}` + : undefined; +} + +async function publishLangSmithSummary({ traceId, runId, experiment, result }) { + const parentRunId = traceUuid(traceId); + if (!langsmithEnabled() || !parentRunId) return; + try { + const { Client } = await import("langsmith"); + const client = new Client({ + apiKey: process.env.LANGSMITH_API_KEY, + apiUrl: process.env.LANGSMITH_ENDPOINT, + workspaceId: process.env.LANGSMITH_WORKSPACE_ID, + omitTracedRuntimeInfo: true, + }); + const now = new Date().toISOString(); + await client.createRun({ + id: randomUUID(), + trace_id: parentRunId, + parent_run_id: parentRunId, + project_name: process.env.LANGSMITH_PROJECT || "slm-training", + name: "agentv.publication", + run_type: "tool", + inputs: { run_id: runId, experiment }, + outputs: { summary: result.summary }, + start_time: now, + end_time: now, + extra: { metadata: { w3c_trace_id: traceId, sdk: "@agentv/core" } }, + }); + await Promise.race([ + client.flush(), + new Promise((resolve) => setTimeout(resolve, 500)), + ]); + } catch (error) { + console.warn(`LangSmith AgentV export failed: ${String(error)}`); + } +} + const specFile = option("--spec"); const outputDir = option("--output-dir"); const experiment = option("--experiment"); const sdkRoot = option("--sdk-root"); +const traceId = optional("--trace-id"); +const runId = optional("--run-id"); const sdkUrl = pathToFileURL( `${sdkRoot}/node_modules/@agentv/core/dist/index.js`, ); @@ -29,6 +82,8 @@ const result = await evaluate({ experiment, }); +await publishLangSmithSummary({ traceId, runId, experiment, result }); + console.log(JSON.stringify({ summary: result.summary, artifacts: result.artifacts, diff --git a/scripts/run_slm299_reachability_audit.py b/scripts/run_slm299_reachability_audit.py index 73dcbf04c..5b65e5c04 100644 --- a/scripts/run_slm299_reachability_audit.py +++ b/scripts/run_slm299_reachability_audit.py @@ -87,7 +87,7 @@ def load_corpora() -> dict[str, list[dict[str, Any]]]: def analyze_record( - record: dict[str, Any], *, max_edits: int, node_budget: int + record: dict[str, Any], *, max_edits: int, node_budget: int, mode: str = "extended" ) -> ReachabilityCase: target = str(record.get("openui") or "") placeholders = record.get("placeholders") or extract_placeholders(target) @@ -97,6 +97,7 @@ def analyze_record( slot_inventory=[str(p) for p in placeholders], max_edits=max_edits, node_budget=node_budget, + mode=mode, ) @@ -185,6 +186,54 @@ def build_x22_evidence_annotations( return annotations +def _summarize_records( + records: list[dict[str, Any]], *, max_edits: int, node_budget: int, mode: str +) -> dict[str, Any]: + cases = [ + ( + str(record.get("id", f"case_{index}")), + analyze_record(record, max_edits=max_edits, node_budget=node_budget, mode=mode), + ) + for index, record in enumerate(records) + ] + return {"status": "ok", **summarize_suite(cases)} + + +def _compare_summaries(v1: dict[str, Any], extended: dict[str, Any]) -> dict[str, Any]: + """Old-vs-extended comparison: verdict flips and action-cost deltas over + matched cases (never a quality claim — reachability only).""" + v1_cases = {c["id"]: c for c in v1.get("cases", [])} + ext_cases = {c["id"]: c for c in extended.get("cases", [])} + deltas: list[int] = [] + flips: list[dict[str, Any]] = [] + for case_id, ext in sorted(ext_cases.items()): + old = v1_cases.get(case_id) + if old is None: + continue + if old["verdict"] != ext["verdict"]: + flips.append( + {"id": case_id, "v1": old["verdict"], "extended": ext["verdict"]} + ) + if old["edit_lower_bound"] is not None and ext["edit_lower_bound"] is not None: + deltas.append(ext["edit_lower_bound"] - old["edit_lower_bound"]) + return { + "n_verdict_flips": len(flips), + "verdict_flips": flips, + "reachable_fraction_v1": v1.get("reachable_fraction"), + "reachable_fraction_extended": extended.get("reachable_fraction"), + "action_cost_delta": ( + { + "min": min(deltas), + "median": statistics.median(deltas), + "max": max(deltas), + "mean": round(statistics.mean(deltas), 4), + } + if deltas + else None + ), + } + + def build_report( corpora: dict[str, list[dict[str, Any]]], *, @@ -192,8 +241,12 @@ def build_report( node_budget: int, generated_at: str, limit: int | None = None, + mode: str = "extended", + compare: bool = False, ) -> dict[str, Any]: suites: dict[str, Any] = {} + suites_v1: dict[str, Any] = {} + comparisons: dict[str, Any] = {} for suite, records in corpora.items(): if not records: suites[suite] = { @@ -201,22 +254,25 @@ def build_report( "n_cases": 0, "reachable_fraction": None, } + if compare: + suites_v1[suite] = dict(suites[suite]) continue if limit is not None: records = records[:limit] - cases = [ - ( - str(record.get("id", f"{suite}_{index}")), - analyze_record(record, max_edits=max_edits, node_budget=node_budget), + suites[suite] = _summarize_records( + records, max_edits=max_edits, node_budget=node_budget, mode=mode + ) + if compare: + suites_v1[suite] = _summarize_records( + records, max_edits=max_edits, node_budget=node_budget, mode="v1" ) - for index, record in enumerate(records) - ] - suites[suite] = {"status": "ok", **summarize_suite(cases)} + comparisons[suite] = _compare_summaries(suites_v1[suite], suites[suite]) payload: dict[str, Any] = { - "schema": "slm299_edit_reachability_audit/v1", + "schema": "slm299_edit_reachability_audit/v2", "experiment_id": EXPERIMENT_ID, "seed_source": DEFAULT_SEED_SOURCE, + "mode": mode, "max_edits": max_edits, "node_budget": node_budget, "generated_at": generated_at, @@ -224,11 +280,15 @@ def build_report( "reachable_fraction is computed over decided cases only; " "UNKNOWN_BUDGET cases are reported separately and are never counted " "as unreachable; suites without a corpus are corpus_unavailable, " - "never zero-reachable." + "never zero-reachable. Reachability is a space-coverage proof, " + "never a model-quality claim." ), "suites": suites, "version_stamp": build_version_stamp(COMPONENT), } + if compare: + payload["suites_v1"] = suites_v1 + payload["old_vs_extended"] = comparisons payload["x22_evidence_annotations"] = build_x22_evidence_annotations( suites, generated_at=generated_at ) @@ -241,9 +301,13 @@ def render_markdown(payload: dict[str, Any]) -> str: "", f"- generated_at: `{payload['generated_at']}`", f"- seed: `{payload['seed_source']}`", + f"- mode: `{payload.get('mode', 'extended')}`", f"- max_edits: {payload['max_edits']}, node_budget: {payload['node_budget']}", f"- verdict policy: {payload['verdict_policy']}", "", + "> Reachability is space coverage, not model quality: no quality claim", + "> follows from these proofs alone.", + "", "## Suite summary", "", "| suite | n | decided | unknown | reachable_fraction | min/med/max edits |", @@ -291,6 +355,30 @@ def render_markdown(payload: dict[str, Any]) -> str: lines += ["", "## X22 evidence annotations (append-only)", ""] for note in payload["x22_evidence_annotations"]: lines.append(f"- `{note['target_doc']}` [{note['suite']}]: {note['annotation']}") + if "old_vs_extended" in payload: + lines += ["", "## Old (v1) vs extended (SLM-305) reachability", ""] + lines.append( + "| suite | reachable v1 | reachable extended | verdict flips | " + "action-cost delta min/med/max |" + ) + lines.append("| --- | --- | --- | --- | --- |") + for suite, comp in payload["old_vs_extended"].items(): + delta = comp["action_cost_delta"] or {} + delta_txt = ( + f"{delta.get('min')}/{delta.get('median')}/{delta.get('max')}" + if delta + else "—" + ) + lines.append( + f"| {suite} | {comp['reachable_fraction_v1']} | " + f"{comp['reachable_fraction_extended']} | " + f"{comp['n_verdict_flips']} | {delta_txt} |" + ) + for suite, comp in payload["old_vs_extended"].items(): + for flip in comp["verdict_flips"]: + lines.append( + f"- flip `{suite}/{flip['id']}`: {flip['v1']} → {flip['extended']}" + ) lines.append("") return "\n".join(lines) @@ -300,6 +388,17 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--max-edits", type=int, default=8) parser.add_argument("--node-budget", type=int, default=800) parser.add_argument("--limit", type=int, default=None, help="per-suite case cap (debug)") + parser.add_argument( + "--mode", + choices=["v1", "extended"], + default="extended", + help="action space to audit (extended is the deployed SLM-305 space)", + ) + parser.add_argument( + "--compare", + action="store_true", + help="also run the v1 space and publish old-vs-extended deltas", + ) parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) args = parser.parse_args(argv) @@ -311,6 +410,8 @@ def main(argv: list[str] | None = None) -> int: node_budget=args.node_budget, generated_at=generated_at, limit=args.limit, + mode=args.mode, + compare=args.compare, ) args.json_out.parent.mkdir(parents=True, exist_ok=True) args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") diff --git a/src/slm_training/evals/agentv.py b/src/slm_training/evals/agentv.py index 56ba94f24..ca59544fc 100644 --- a/src/slm_training/evals/agentv.py +++ b/src/slm_training/evals/agentv.py @@ -101,8 +101,7 @@ def publish_agentv_evaluation( spec_path.write_text("\n".join(rows) + "\n", encoding="utf-8") runner, runtime_root = _agentv_runtime(repo_root) - completed = subprocess.run( - [ + command = [ "node", str(runner), "--spec", @@ -113,7 +112,12 @@ def publish_agentv_evaluation( slug, "--sdk-root", str(runtime_root), - ], + ] + trace_id = _run_trace_id(Path(run_dir)) + if trace_id is not None: + command.extend(("--trace-id", trace_id, "--run-id", Path(run_dir).name)) + completed = subprocess.run( + command, cwd=runtime_root, check=False, capture_output=True, @@ -182,6 +186,18 @@ def _read_agentv_criterion_results(published: dict[str, Any]) -> list[dict[str, return results +def _run_trace_id(run_dir: Path) -> str | None: + """Return the valid W3C trace ID associated with a local run, if present.""" + path = run_dir / "trace.json" + if not path.is_file(): + return None + try: + trace_id = str(json.loads(path.read_text(encoding="utf-8")).get("trace_id", "")) + except (OSError, json.JSONDecodeError): + return None + return trace_id if re.fullmatch(r"[0-9a-f]{32}", trace_id) else None + + def _stamp_agentv_artifacts( output_dir: Path, version_stamp: dict[str, Any] ) -> None: diff --git a/src/slm_training/harnesses/experiments/slm299_edit_reachability.py b/src/slm_training/harnesses/experiments/slm299_edit_reachability.py index 537776b07..7d666819e 100644 --- a/src/slm_training/harnesses/experiments/slm299_edit_reachability.py +++ b/src/slm_training/harnesses/experiments/slm299_edit_reachability.py @@ -33,16 +33,26 @@ from slm_training.dsl.parser import validate from slm_training.models.tree_edit_diffusion import ( ACTION_ADD, + ACTION_ADD_CONTAINER, + ACTION_BIND_PLACEHOLDER, + ACTION_INSERT_STATEMENT, + ACTION_INSERT_SUBTREE, ACTION_REMOVE, + ACTION_REMOVE_CONTAINER, ACTION_REPLACE, + ACTION_REPLACE_STATEMENT, + ACTION_REPLACE_SUBTREE, CONTAINER_COMPONENTS, + CONTAINER_RESTS, LEAF_COMPONENTS, MAX_SLOTS, + V05_TEMPLATES, Edit, Statement, TreeEditSpace, parse_statements, render_statements, + v05_template_index, ) __all__ = [ @@ -111,9 +121,9 @@ def add_container_action( components: Sequence[str] = CONTAINER_COMPONENTS, ) -> ExtraAction: """Synthetic ``ADD_CONTAINER``: append a fresh empty container statement - and reference it from an existing container. Does **not** exist in the - real X22 space (ADD creates leaves only); used to prove that a target is - unreachable *because* the space cannot create containers. + and reference it from an existing container. Retired what-if lane: the + extended (SLM-305) space has the real ``ACTION_ADD_CONTAINER``, so this + synthetic action is only meaningful in ``mode="v1"`` analyses. """ def _generate( @@ -201,13 +211,22 @@ def _normalize_inventory(slot_inventory: Sequence[str]) -> list[str]: return inventory[:MAX_SLOTS] -def _is_unsupported_pack_feature(target_source: str) -> bool: +def _is_unsupported_pack_feature( + target_source: str, *, extended: bool = False +) -> bool: if any(marker in target_source for marker in _V05_TEXT_MARKERS): return True for line in target_source.splitlines(): line = line.strip() if line and _V05_LINE_RE.match(line): - return True + if not extended: + return True + # SLM-305: the extended space models V0.5 statements through the + # canonical-template statement actions; a V0.5 line that does not + # instantiate a canonical template is still unsupported. + stmt = parse_statements(line) + if stmt is None or len(stmt) != 1 or v05_template_index(stmt[0]) is None: + return True return False @@ -260,24 +279,41 @@ def _check_invariants( inventory: list[str], space: TreeEditSpace, capabilities: frozenset[str] = frozenset(), + *, + extended: bool = False, ) -> str | None: - """Structural impossibility proofs over the EXACT X22 action set. - - REPLACE preserves container-ness, arity, and the container's enum/direction - arg; ADD creates leaves only and binds only inventory slots; REMOVE deletes - leaves only. Anything the target needs beyond that is proven unreachable. + """Structural impossibility proofs over the EXACT action set. + + v1 space: REPLACE preserves container-ness, arity, and the container's + enum/direction arg; ADD creates leaves only and binds only inventory + slots; REMOVE deletes leaves only. Extended (SLM-305) space: ADD_CONTAINER + / INSERT_SUBTREE mint containers carrying one of ``CONTAINER_RESTS`` and + bind only inventory slots, REPLACE_SUBTREE / BIND_PLACEHOLDER rebind + leaves only to + inventory slots, and the V0.5 statement actions mint only canonical + templates — the invariant reasons fire only when the corresponding REAL + action of the analyzed mode is absent. """ seed_containers = [s for s in seed if s.has_list] target_containers = [s for s in target if s.has_list] - target_leaves = [s for s in target if not s.has_list] - - # No action creates a container (ADD is leaf-only). + # V0.5 canonical-template statements are produced by the statement actions + # in the extended space; they are not UI leaves/containers. + target_leaves = [ + s + for s in target + if not s.has_list and not (extended and v05_template_index(s) is not None) + ] + + # No action creates a container in the v1 space (ADD is leaf-only); the + # extended space has the real ADD_CONTAINER / INSERT_SUBTREE. if ( len(target_containers) > len(seed_containers) and "container_add" not in capabilities ): return REASON_NEEDS_CONTAINER_ADD - # No action removes a container (REMOVE is leaf-only). + # No action removes a container (REMOVE is leaf-only); REMOVE_CONTAINER + # can only remove non-root containers, and both seed and target keep root, + # so this invariant holds in every mode. if len(target_containers) < len(seed_containers): return REASON_NEEDS_CONTAINER_REMOVE @@ -289,9 +325,9 @@ def _check_invariants( if stmt.comp not in LEAF_COMPONENTS or stmt.comp not in known: return REASON_UNSUPPORTED_COMPONENT - # ADD binds only inventory slots; REPLACE cannot change a leaf's bound - # slot, and the seed carries no leaves. So every target leaf slot must - # come from the prompt inventory. + # ADD / INSERT_SUBTREE / REPLACE_SUBTREE / BIND_PLACEHOLDER bind only + # inventory slots, and the seed carries no leaves. So every target leaf + # slot must come from the prompt inventory in every mode. for stmt in target_leaves: slot = _leaf_slot(stmt.rest) if slot is None: @@ -301,14 +337,24 @@ def _check_invariants( return REASON_NEEDS_SLOT_REBIND # REPLACE preserves the container's raw enum/direction arg text (rest); - # no real action edits it. Every target container must therefore carry - # the seed container's rest. With a synthetic container-creating action - # (which mints containers carrying the seed's rest), the same per-container - # rule applies; without it the multisets must match exactly. + # no real action edits it. Containers minted by the container-creating + # actions carry exactly one of CONTAINER_RESTS. Every target container + # rest must therefore be a seed rest or (with container_add) a candidate + # mint rest; without any container-creating action the multisets must + # match exactly. seed_rests = sorted(s.rest for s in seed_containers) target_rests = sorted(s.rest for s in target_containers) + # Root can never be removed or re-minted, and REPLACE preserves rest: + # the target root's rest must equal the seed root's rest in every mode. + seed_root_rest = next( + (s.rest for s in seed_containers if s.name == "root"), None + ) + for stmt in target_containers: + if stmt.name == "root" and stmt.rest != seed_root_rest: + return REASON_NEEDS_DIRECTION_CHANGE if "container_add" in capabilities: - if any(rest not in seed_rests for rest in target_rests): + allowed = set(seed_rests) | set(CONTAINER_RESTS) + if any(rest not in allowed for rest in target_rests): return REASON_NEEDS_DIRECTION_CHANGE elif seed_rests != target_rests: return REASON_NEEDS_DIRECTION_CHANGE @@ -320,17 +366,45 @@ def _enumerate_children( space: TreeEditSpace, statements: list[Statement], inventory: list[str], + *, + mode: str = "extended", + visited: set[str] | None = None, ) -> list[tuple[list[Statement], dict[str, Any]]]: - """All one-edit successors under the REAL X22 action set, applied through - ``TreeEditSpace.apply`` so preconditions and parser re-validation are the - deployed ones by construction.""" + """All one-edit successors under the REAL action set of ``mode``, applied + through ``TreeEditSpace.apply`` so preconditions and parser re-validation + are the deployed ones by construction. ``v1`` enumerates the original + REPLACE/ADD/REMOVE set; ``extended`` (SLM-305) adds the container, + subtree, V0.5-statement, and placeholder-binding actions. Deterministic: + same state and mode always yield the same enumeration order. + + ``visited`` (canonical-key set) is a pure search-efficiency hook: states + already keyed are rejected pre-validation inside ``apply``; it never + changes WHICH distinct states are reachable.""" children: list[tuple[list[Statement], dict[str, Any]]] = [] n_comp = len(space.components) n_slots = min(len(inventory), MAX_SLOTS) + leaf_comp_idxs = [ + i for i, c in enumerate(space.components) if c in LEAF_COMPONENTS + ] + container_comp_idxs = [ + i for i, c in enumerate(space.components) if c in CONTAINER_COMPONENTS + ] + pre = None + if visited is not None: + pre = lambda working: _canonical_key(working) not in visited # noqa: E731 + by_name = {s.name: s for s in statements} + + def _replace_subtree_ok(stmt: Statement) -> bool: + if not stmt.has_list or len(stmt.children) != 1: + return False + leaf = by_name.get(stmt.children[0]) + return leaf is not None and not leaf.has_list + for stmt_idx in range(len(statements)): + stmt = statements[stmt_idx] for comp_idx in range(n_comp): edit = Edit(ACTION_REPLACE, stmt_idx, comp_idx) - nxt = space.apply(statements, edit, inventory) + nxt = space.apply(statements, edit, inventory, pre) if nxt is not None: children.append( ( @@ -344,7 +418,7 @@ def _enumerate_children( ) for slot_idx in range(n_slots): edit = Edit(ACTION_ADD, stmt_idx, comp_idx, slot_idx) - nxt = space.apply(statements, edit, inventory) + nxt = space.apply(statements, edit, inventory, pre) if nxt is not None: children.append( ( @@ -357,10 +431,124 @@ def _enumerate_children( }, ) ) + if mode == "v1": + edit = Edit(ACTION_REMOVE, stmt_idx) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append((nxt, {"action": "REMOVE", "stmt": stmt_idx})) + continue + # SLM-305 extended real actions. + if stmt.has_list: + for comp_idx in container_comp_idxs: + for rest_idx in range(len(CONTAINER_RESTS)): + edit = Edit(ACTION_ADD_CONTAINER, stmt_idx, comp_idx, + target=rest_idx) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "ADD_CONTAINER", + "stmt": stmt_idx, + "comp": space.components[comp_idx], + "rest": CONTAINER_RESTS[rest_idx], + }, + ) + ) + for comp_idx in container_comp_idxs: + for slot_idx in range(n_slots): + for payload in leaf_comp_idxs: + for rest_idx in range(len(CONTAINER_RESTS)): + edit = Edit( + ACTION_INSERT_SUBTREE, stmt_idx, comp_idx, slot_idx, + target=rest_idx, payload=payload, + ) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "INSERT_SUBTREE", + "stmt": stmt_idx, + "comp": space.components[comp_idx], + "slot": inventory[slot_idx], + "leaf_comp": space.components[payload], + "rest": CONTAINER_RESTS[rest_idx], + }, + ) + ) + if _replace_subtree_ok(stmt): + for slot_idx in range(n_slots): + for payload in leaf_comp_idxs: + edit = Edit( + ACTION_REPLACE_SUBTREE, stmt_idx, slot=slot_idx, + payload=payload, + ) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "REPLACE_SUBTREE", + "stmt": stmt_idx, + "slot": inventory[slot_idx], + "leaf_comp": space.components[payload], + }, + ) + ) + else: + for slot_idx in range(n_slots): + edit = Edit(ACTION_BIND_PLACEHOLDER, stmt_idx, slot=slot_idx) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "BIND_PLACEHOLDER", + "stmt": stmt_idx, + "slot": inventory[slot_idx], + }, + ) + ) + edit = Edit(ACTION_REMOVE_CONTAINER, stmt_idx) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + (nxt, {"action": "REMOVE_CONTAINER", "stmt": stmt_idx}) + ) edit = Edit(ACTION_REMOVE, stmt_idx) - nxt = space.apply(statements, edit, inventory) + nxt = space.apply(statements, edit, inventory, pre) if nxt is not None: children.append((nxt, {"action": "REMOVE", "stmt": stmt_idx})) + for payload in range(len(V05_TEMPLATES)): + edit = Edit(ACTION_REPLACE_STATEMENT, stmt_idx, payload=payload) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "REPLACE_STATEMENT", + "stmt": stmt_idx, + "template": payload, + }, + ) + ) + if mode != "v1": + for payload in range(len(V05_TEMPLATES)): + edit = Edit(ACTION_INSERT_STATEMENT, payload=payload) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + {"action": "INSERT_STATEMENT", "template": payload}, + ) + ) return children @@ -372,14 +560,19 @@ def analyze_reachability( max_edits: int = 8, extra_actions: Sequence[ExtraAction] = (), node_budget: int = 800, + mode: str = "extended", ) -> ReachabilityCase: """Prove (or honestly fail to prove) reachability of ``target_source`` - from ``seed_source`` under the real X22 tree-edit space. + from ``seed_source`` under the real tree-edit space of ``mode``. - ``extra_actions`` are hypothetical transitions (what-if analysis only); - when any appear on a found path the case is marked in ``details`` so the - proof is never confused with the real space. + ``mode="v1"`` analyzes the original SLM-299 action set (REPLACE / ADD / + REMOVE); ``mode="extended"`` (default, SLM-305) analyzes the deployed + extended edit language. ``extra_actions`` are hypothetical transitions + (what-if analysis only); when any appear on a found path the case is + marked in ``details`` so the proof is never confused with the real space. """ + if mode not in {"v1", "extended"}: + raise ValueError(f"unknown reachability mode {mode!r}") space = _shared_space() inventory = _normalize_inventory(slot_inventory) @@ -396,9 +589,10 @@ def analyze_reachability( "node_budget": node_budget, "inventory_size": len(inventory), "extra_actions": [a.name for a in extra_actions], + "mode": mode, } - if _is_unsupported_pack_feature(target_source): + if _is_unsupported_pack_feature(target_source, extended=(mode == "extended")): return ReachabilityCase( verdict=Verdict.PROVEN_UNREACHABLE, reason_code=REASON_UNSUPPORTED_PACK_FEATURE, @@ -423,7 +617,14 @@ def analyze_reachability( capabilities = frozenset().union( *(a.capabilities for a in extra_actions) ) if extra_actions else frozenset() - fired = _check_invariants(target, seed, inventory, space, capabilities) + if mode == "extended": + # ADD_CONTAINER / INSERT_SUBTREE are REAL actions in the extended + # space, so the container_add invariant must not fire; the synthetic + # add_container_action what-if lane is retired for this space. + capabilities = capabilities | frozenset({"container_add"}) + fired = _check_invariants( + target, seed, inventory, space, capabilities, extended=(mode == "extended") + ) if fired is not None: return ReachabilityCase( verdict=Verdict.PROVEN_UNREACHABLE, @@ -456,7 +657,9 @@ def analyze_reachability( if depth >= max_edits: live_at_budget = True continue - children = _enumerate_children(space, statements, inventory) + children = _enumerate_children( + space, statements, inventory, mode=mode, visited=visited + ) for extra in extra_actions: for nxt, action in extra.generate(statements, inventory): children.append((nxt, {**action, "synthetic": True})) diff --git a/src/slm_training/harnesses/model_build/eval_runner.py b/src/slm_training/harnesses/model_build/eval_runner.py index 645808776..9ec6ba4fd 100644 --- a/src/slm_training/harnesses/model_build/eval_runner.py +++ b/src/slm_training/harnesses/model_build/eval_runner.py @@ -48,6 +48,13 @@ from slm_training.versioning import component_version _COMPONENT_RE = re.compile(r"\b([A-Z][A-Za-z0-9]*)\s*\(") +_LANGSMITH_METRIC_KEYS = ( + "n", + "parse_rate", + "placeholder_fidelity", + "structural_similarity", + "reward_score", +) def _evaluation_version_components(config: ModelBuildConfig) -> tuple[str, ...]: @@ -62,6 +69,43 @@ def _evaluation_version_components(config: ModelBuildConfig) -> tuple[str, ...]: return components + (("model.twotower",) if config.model_name == "twotower" else ()) +def _record_langsmith_evaluation(config, *, suites: dict[str, dict], scoreboard: dict) -> None: + """Publish only aggregate evaluation data to the active summary trace.""" + from slm_training.runtime.telemetry import current_trace + + trace = current_trace() + if trace is None: + return + summary = { + suite: { + key: metrics[key] + for key in _LANGSMITH_METRIC_KEYS + if key in metrics + } + for suite, metrics in suites.items() + } + trace.record_summary( + "evaluation.summary", + inputs={"run_id": config.run_id, "suites": sorted(suites)}, + outputs={ + "suites": summary, + "gates": scoreboard.get("gates"), + "agentv": scoreboard.get("agentv"), + }, + metadata={ + key: scoreboard.get(key) + for key in ( + "run_class", + "checkpoint_sha256", + "eval_data_manifest_sha", + "code_git_sha", + "version_stamp", + ) + if scoreboard.get(key) is not None + }, + ) + + def _annotate_decode_trace_records( stats: object, records: list[ExampleRecord], @@ -1706,6 +1750,19 @@ def mean(key: str) -> float: suite_path.write_text(payload, encoding="utf-8") if config.suite == "smoke": (run_dir / "eval.json").write_text(payload, encoding="utf-8") + if publish_agentv: + _record_langsmith_evaluation( + config, + suites={config.suite: metrics}, + scoreboard={ + "run_class": config.run_class, + "checkpoint_sha256": metrics.get("checkpoint_sha256"), + "eval_data_manifest_sha": metrics.get("eval_data_manifest_sha"), + "code_git_sha": metrics.get("code_git_sha"), + "version_stamp": metrics["version_stamp"], + "agentv": metrics.get("agentv"), + }, + ) return metrics @@ -1830,4 +1887,5 @@ def evaluate_suites( ) } path.write_text(json.dumps(scoreboard, indent=2) + "\n", encoding="utf-8") + _record_langsmith_evaluation(config, suites=board, scoreboard=scoreboard) return scoreboard diff --git a/src/slm_training/models/checkpoint_migrate.py b/src/slm_training/models/checkpoint_migrate.py index 5342cef2d..46e738705 100644 --- a/src/slm_training/models/checkpoint_migrate.py +++ b/src/slm_training/models/checkpoint_migrate.py @@ -293,3 +293,97 @@ def migrate_to_shared_recursive_denoiser( report_path = new_path.with_suffix(".migrate.json") report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") return report + + +def migrate_tree_edit_checkpoint( + *, + source_checkpoint: Path | str, + output_checkpoint: Path | str, + device: str = "cpu", +) -> dict: + """Warm-start a tree-edit-diffusion format-1 checkpoint to format 2. + + SLM-305 grew ``action_head`` from 4 to ``N_ACTIONS`` rows (extended edit + language). All same-shape tensors are copied verbatim; the old action-head + rows are preserved in the first rows of the new head and the new action + rows stay randomly initialized. A ``.migrate.json`` report is written next + to the output checkpoint. + """ + from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + ) + + source_checkpoint = Path(source_checkpoint) + output_checkpoint = Path(output_checkpoint) + payload = torch.load(source_checkpoint, map_location=device, weights_only=False) + if payload.get("kind") != "tree_edit_diffusion": + raise ValueError( + f"checkpoint kind {payload.get('kind')!r} is not tree_edit_diffusion" + ) + source_format = int(payload.get("format_version") or 1) + if source_format >= TreeEditDiffusionModel.CHECKPOINT_FORMAT: + raise ValueError( + f"tree-edit checkpoint is already format {source_format}; " + "migration only upgrades older formats" + ) + tokenizer_path = source_checkpoint.with_suffix(".tokenizer.json") + if not tokenizer_path.exists(): + raise FileNotFoundError( + f"missing tokenizer next to checkpoint: {tokenizer_path}" + ) + tokenizer = OpenUITokenizer.load(tokenizer_path, allow_legacy=True) + raw_config = dict(payload.get("config") or {}) + valid = set(TreeEditDiffusionConfig.__dataclass_fields__) + config = TreeEditDiffusionConfig( + **{key: value for key, value in raw_config.items() if key in valid} + ) + model = TreeEditDiffusionModel(tokenizer, config=config, device=device) + old_state = payload.get("state_dict") or {} + new_state = model.state_dict() + copied_keys: list[str] = [] + skipped_old_keys: list[str] = [] + initialized_keys: list[str] = [] + preserved_action_rows = 0 + for key, tensor in old_state.items(): + if key in new_state and new_state[key].shape == tensor.shape: + new_state[key] = tensor + copied_keys.append(key) + elif ( + key == "policy.action_head.weight" + and key in new_state + and tensor.shape[1] == new_state[key].shape[1] + and tensor.shape[0] <= new_state[key].shape[0] + ): + new_state[key][: tensor.shape[0]] = tensor + copied_keys.append(key) + preserved_action_rows = int(tensor.shape[0]) + elif ( + key == "policy.action_head.bias" + and key in new_state + and tensor.shape[0] <= new_state[key].shape[0] + ): + new_state[key][: tensor.shape[0]] = tensor + copied_keys.append(key) + else: + skipped_old_keys.append(key) + for key in new_state: + if key not in copied_keys: + initialized_keys.append(key) + model.load_state_dict(new_state, strict=True) + model.save(output_checkpoint) + report = { + "source_checkpoint": str(source_checkpoint), + "output_checkpoint": str(output_checkpoint), + "source_format_version": source_format, + "output_format_version": TreeEditDiffusionModel.CHECKPOINT_FORMAT, + "warm_start_only": True, + "preserved_action_head_rows": preserved_action_rows, + "copied_keys": copied_keys, + "skipped_old_keys": skipped_old_keys, + "initialized_keys": initialized_keys, + } + output_checkpoint.with_suffix(".migrate.json").write_text( + json.dumps(report, indent=2) + "\n", encoding="utf-8" + ) + return report diff --git a/src/slm_training/models/tree_edit_diffusion.py b/src/slm_training/models/tree_edit_diffusion.py index 26e032cf8..13ffc430d 100644 --- a/src/slm_training/models/tree_edit_diffusion.py +++ b/src/slm_training/models/tree_edit_diffusion.py @@ -23,14 +23,15 @@ import random import re from dataclasses import asdict, dataclass +from functools import lru_cache from pathlib import Path -from typing import Any +from typing import Any, Callable import torch import torch.nn as nn import torch.nn.functional as F -from slm_training.dsl.parser import validate +from slm_training.dsl.parser import validate, validate_output from slm_training.dsl.placeholders import extract_placeholders from slm_training.dsl.schema import ExampleRecord from slm_training.harnesses.model_build.plugin import GenerationRequest @@ -49,7 +50,18 @@ ACTION_REPLACE = 1 # swap the component type of one statement ACTION_ADD = 2 # add a fresh leaf statement + reference it from a container ACTION_REMOVE = 3 # remove one leaf statement and its references -N_ACTIONS = 4 +# SLM-305 (LAR2-01) extended valid-state edit language. Every new action is +# bounded, deterministic, validity-preserving (re-validated through the real +# parser before acceptance), and invertible; preconditions are documented at +# each branch of ``TreeEditSpace.apply``. +ACTION_ADD_CONTAINER = 4 # insert an empty container into a container's children +ACTION_REMOVE_CONTAINER = 5 # safe inverse: remove an empty/leaf-only container subtree +ACTION_INSERT_SUBTREE = 6 # transactional: declare container+leaf subtree and reference it +ACTION_REPLACE_SUBTREE = 7 # replace a canonical one-leaf container subtree's leaf +ACTION_INSERT_STATEMENT = 8 # insert a canonical V0.5 state/query/mutation statement +ACTION_REPLACE_STATEMENT = 9 # swap one canonical V0.5 statement for another +ACTION_BIND_PLACEHOLDER = 10 # (re)bind a leaf's slot to an inventory placeholder +N_ACTIONS = 11 MAX_STMTS = 24 MAX_SLOTS = 16 @@ -60,6 +72,40 @@ LEAF_COMPONENTS = ("TextContent", "Button", "Image", "TextInput") CONTAINER_COMPONENTS = ("Stack", "Card", "Form") +# Canonical rest texts for containers minted by ADD_CONTAINER / INSERT_SUBTREE. +# Fixed candidate set (indexed by ``Edit.target``) so minted containers are +# deterministic and the reachability invariants can reason about them exactly. +CONTAINER_RESTS: tuple[str, ...] = (', "column"', "") +CONTAINER_REST = CONTAINER_RESTS[0] + +# V0.5 statement component names (state/query/mutation/action pack forms). +V05_COMPONENTS = ("Query", "Mutation", "Action", "State", "Resource") + +# Bounded canonical V0.5 statement templates: (component, canonical arg text). +# Construction is canonical-AST-backed: each inserted/replaced line is built +# from this structured spec and fragment-validated through the canonical +# grammar (``validate_output(..., kind="statement")``) before acceptance — +# never regex string surgery on existing program text. +V05_TEMPLATES: tuple[tuple[str, str], ...] = ( + ("Query", '"tool", {arg: $x}, {default: []}, 15'), + ("Mutation", '"tool", {arg: $x}'), +) + + +def v05_template_index(stmt: Statement) -> int | None: + """Index of the canonical V0.5 template ``stmt`` instantiates, else None. + + Only canonical-template statements are REPLACE_STATEMENT-editable, so the + inverse edit (restore the old template) is always expressible. + """ + if stmt.has_list: + return None + rest = stmt.rest.strip() + for index, (comp, args) in enumerate(V05_TEMPLATES): + if stmt.comp == comp and rest == args: + return index + return None + _STMT_RE = re.compile(r"^(?P\w+)\s*=\s*(?P\w+)\((?P.*)\)\s*$") @@ -101,7 +147,14 @@ def render(self) -> str: def parse_statements(source: str) -> list[Statement] | None: """Structural parse of a canonical program; None when a line defies the - `name = Comp(...)` shape (those programs are skipped, never mutated).""" + `name = Comp(...)` shape (those programs are skipped, never mutated). + + V0.5 statement lines (Query/Mutation/Action/State/Resource) are owned by + the canonical grammar: the line is fragment-validated through + ``validate_output(..., kind="statement")`` rather than the regex alone. + UI statement lines keep the legacy structural split so + ``Statement.render()`` stays byte-stable for existing fixture programs. + """ statements: list[Statement] = [] for line in source.splitlines(): line = line.strip() @@ -110,6 +163,11 @@ def parse_statements(source: str) -> list[Statement] | None: match = _STMT_RE.match(line) if match is None: return None + if match.group("comp") in V05_COMPONENTS: + try: + validate_output(line, kind="statement") + except Exception: # noqa: BLE001 + return None args = match.group("args") if args.startswith("["): depth = 0 @@ -152,6 +210,7 @@ def render_statements(statements: list[Statement]) -> str: return "\n".join(stmt.render() for stmt in statements) +@lru_cache(maxsize=65536) def _is_valid(source: str) -> bool: try: validate(source) @@ -162,12 +221,21 @@ def _is_valid(source: str) -> bool: @dataclass(frozen=True) class Edit: - """One bounded edit: action + statement index + component + slot.""" + """One bounded edit: action + statement index + component + slot. + + SLM-305: ``target``/``payload`` are NEW DEFAULTED fields only, so old + pickles and comparisons keep working. ``payload`` carries the leaf + component index (INSERT_SUBTREE / REPLACE_SUBTREE) or the canonical V0.5 + template index (INSERT_STATEMENT / REPLACE_STATEMENT); ``target`` is + reserved for secondary statement addressing. + """ action: int stmt: int = 0 comp: int = 0 slot: int = 0 + target: int = 0 + payload: int = 0 class TreeEditSpace: @@ -192,9 +260,38 @@ def fresh_name(self, statements: list[Statement]) -> str: return name return f"n{len(statements)}x" + def fresh_v05_name(self, statements: list[Statement], comp: str) -> str: + """Fresh V0.5 statement name with the conventional pack prefix.""" + prefix = {"Query": "q", "Mutation": "m"}.get(comp, "r") + taken = {s.name for s in statements} + for i in range(len(statements) + 8): + name = f"{prefix}{i}" + if name not in taken: + return name + return f"{prefix}{len(statements)}x" + + @staticmethod + def _placeholder(inventory: list[str], slot: int) -> str: + placeholder = inventory[slot] + if not placeholder.startswith(":"): + placeholder = f":{placeholder}" + return placeholder + def apply( - self, statements: list[Statement], edit: Edit, inventory: list[str] + self, + statements: list[Statement], + edit: Edit, + inventory: list[str], + pre_validate: Callable[[list[Statement]], bool] | None = None, ) -> list[Statement] | None: + """Apply one edit; None when inapplicable or invalid (fail closed). + + ``pre_validate`` is an optional cheap rejection hook invoked on the + mutated statement list just before parser re-validation (used by the + reachability analyzer to skip already-visited states). It can only + reject, never accept: every accepted state is still re-validated + through the real parser. + """ if edit.action == ACTION_STOP: return [Statement(**vars(s)) for s in statements] working = [ @@ -248,11 +345,188 @@ def apply( if target.name in other.children: other.children = [c for c in other.children if c != target.name] referenced = True - if not referenced: + if not referenced and target.comp not in V05_COMPONENTS: + # Unreferenced UI leaves stay immutable (old behavior); V0.5 + # pack statements are unreferenced by construction and are + # removable (inverse of INSERT_STATEMENT). return None working = [s for s in working if s.name != target.name] + elif edit.action == ACTION_ADD_CONTAINER: + # Preconditions: parent is a container, MAX_STMTS bound, comp is a + # container. The minted container starts EMPTY (leaf-only subtree) + # so REMOVE_CONTAINER is an exact safe inverse. + if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): + return None + if len(working) >= MAX_STMTS: + return None + parent = working[edit.stmt] + comp = self.components[edit.comp] + if not parent.has_list or comp not in CONTAINER_COMPONENTS: + return None + if not (0 <= edit.target < len(CONTAINER_RESTS)): + return None + name = self.fresh_name(working) + parent.children.append(name) + working.append( + Statement( + name=name, comp=comp, children=[], + rest=CONTAINER_RESTS[edit.target], has_list=True, + ) + ) + elif edit.action == ACTION_REMOVE_CONTAINER: + # Safe inverse of ADD_CONTAINER / INSERT_SUBTREE. Preconditions: + # target is a non-root container, referenced from some parent, and + # its subtree is leaf-only (no nested containers) — exactly the + # shapes the container-creating actions mint, so removal restores + # the prior state exactly. Leaf children are dropped with it. + if not (0 <= edit.stmt < len(working)): + return None + target = working[edit.stmt] + if not target.has_list or target.name == "root": + return None + by_name = {s.name: s for s in working} + if any( + by_name.get(child) is None or by_name[child].has_list + for child in target.children + ): + return None + if not any(target.name in other.children for other in working): + return None + drop = {target.name, *target.children} + working = [ + Statement( + s.name, + s.comp, + [c for c in s.children if c not in drop], + s.rest, + s.has_list, + ) + for s in working + if s.name not in drop + ] + elif edit.action == ACTION_INSERT_SUBTREE: + # Transactional declare-plus-reference: mint a small canonical + # subtree (container root + one leaf child bound to an inventory + # slot) and reference the root from an existing container, all + # re-validated as one step. Preconditions: parent is a container, + # comp is a container, payload indexes a leaf component, slot is + # in inventory, MAX_STMTS bound. Inverse: REMOVE_CONTAINER. + if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): + return None + if len(working) + 2 > MAX_STMTS: + return None + parent = working[edit.stmt] + root_comp = self.components[edit.comp] + if not parent.has_list or root_comp not in CONTAINER_COMPONENTS: + return None + if not (0 <= edit.payload < len(self.components)): + return None + leaf_comp = self.components[edit.payload] + if leaf_comp not in LEAF_COMPONENTS: + return None + if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): + return None + if not (0 <= edit.target < len(CONTAINER_RESTS)): + return None + placeholder = self._placeholder(inventory, edit.slot) + cname = self.fresh_name(working) + lname = self.fresh_name( + [*working, Statement(cname, root_comp, [], CONTAINER_REST, True)] + ) + parent.children.append(cname) + working.append( + Statement(cname, root_comp, [lname], CONTAINER_RESTS[edit.target], True) + ) + working.append( + Statement( + lname, leaf_comp, [], + json.dumps(placeholder, ensure_ascii=False), False, + ) + ) + elif edit.action == ACTION_REPLACE_SUBTREE: + # Replace the leaf of a canonical one-leaf container subtree + # (same root kind, so the subtree shape is preserved). + # Preconditions: target container has exactly one child which is a + # leaf, payload indexes a leaf component, slot is in inventory. + # The small-canonical-subtree precondition keeps the inverse + # (restore old leaf comp + slot) expressible as the same action. + if not (0 <= edit.stmt < len(working)): + return None + target = working[edit.stmt] + if not target.has_list or len(target.children) != 1: + return None + by_name = {s.name: s for s in working} + leaf = by_name.get(target.children[0]) + if leaf is None or leaf.has_list: + return None + if not (0 <= edit.payload < len(self.components)): + return None + leaf_comp = self.components[edit.payload] + if leaf_comp not in LEAF_COMPONENTS: + return None + if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): + return None + placeholder = self._placeholder(inventory, edit.slot) + leaf.comp = leaf_comp + leaf.rest = json.dumps(placeholder, ensure_ascii=False) + elif edit.action == ACTION_INSERT_STATEMENT: + # Insert a canonical V0.5 state/query/mutation statement built + # from the structured template spec and fragment-validated through + # the canonical grammar (never regex string surgery). + # Preconditions: MAX_STMTS bound, payload indexes V05_TEMPLATES. + # Inverse: REMOVE (pack statements are unreferenced). + if len(working) >= MAX_STMTS: + return None + if not (0 <= edit.payload < len(V05_TEMPLATES)): + return None + comp, args = V05_TEMPLATES[edit.payload] + candidate = Statement(self.fresh_v05_name(working, comp), comp, [], args, False) + try: + validate_output(candidate.render(), kind="statement") + except Exception: # noqa: BLE001 + return None + working.append(candidate) + elif edit.action == ACTION_REPLACE_STATEMENT: + # Swap one canonical V0.5 statement for another template. + # Preconditions: target instantiates a known canonical template + # (so the inverse — restore the old template — is expressible), + # payload indexes V05_TEMPLATES, and the swap is a real change. + if not (0 <= edit.stmt < len(working)): + return None + target = working[edit.stmt] + if v05_template_index(target) is None: + return None + if not (0 <= edit.payload < len(V05_TEMPLATES)): + return None + comp, args = V05_TEMPLATES[edit.payload] + if target.comp == comp and target.rest.strip() == args: + return None + candidate = Statement(target.name, comp, [], args, False) + try: + validate_output(candidate.render(), kind="statement") + except Exception: # noqa: BLE001 + return None + working[edit.stmt] = candidate + elif edit.action == ACTION_BIND_PLACEHOLDER: + # Transactional declaration-plus-reference: (re)bind a leaf's slot + # to an inventory placeholder. Preconditions: target is a UI leaf + # (non-root, non-container, leaf component), slot in inventory. + # Inverse: BIND_PLACEHOLDER with the old slot index. + if not (0 <= edit.stmt < len(working)): + return None + target = working[edit.stmt] + if target.has_list or target.name == "root": + return None + if target.comp not in LEAF_COMPONENTS: + return None + if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): + return None + placeholder = self._placeholder(inventory, edit.slot) + target.rest = json.dumps(placeholder, ensure_ascii=False) else: return None + if pre_validate is not None and not pre_validate(working): + return None rendered = render_statements(working) if not _is_valid(rendered): return None @@ -267,7 +541,20 @@ def sample_mutation( """One random validity-preserving mutation and the *inverse* edit (the supervised repair step) — Kapur's forward process.""" for _ in range(12): - kind = rng.choice((ACTION_REPLACE, ACTION_ADD, ACTION_REMOVE)) + kind = rng.choice( + ( + ACTION_REPLACE, + ACTION_ADD, + ACTION_REMOVE, + ACTION_ADD_CONTAINER, + ACTION_REMOVE_CONTAINER, + ACTION_INSERT_SUBTREE, + ACTION_REPLACE_SUBTREE, + ACTION_INSERT_STATEMENT, + ACTION_REPLACE_STATEMENT, + ACTION_BIND_PLACEHOLDER, + ) + ) if kind == ACTION_REPLACE: idx = rng.randrange(len(statements)) stmt = statements[idx] @@ -304,6 +591,210 @@ def sample_mutation( continue inverse = Edit(ACTION_REMOVE, len(mutated) - 1) return mutated, inverse + if kind == ACTION_ADD_CONTAINER: + # Mutation = empty container under a container; inverse = + # REMOVE_CONTAINER (exact, the minted subtree is empty). + parents = [i for i, s in enumerate(statements) if s.has_list] + if not parents or len(statements) >= MAX_STMTS: + continue + parent_idx = rng.choice(parents) + comp = rng.choice( + [c for c in CONTAINER_COMPONENTS if c in self.comp_index] + ) + mutation = Edit( + ACTION_ADD_CONTAINER, parent_idx, self.comp_index[comp], + target=rng.randrange(len(CONTAINER_RESTS)), + ) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_REMOVE_CONTAINER, len(mutated) - 1) + return mutated, inverse + if kind == ACTION_REMOVE_CONTAINER: + # Mutation = remove an empty/leaf-only container subtree; + # inverse = ADD_CONTAINER (empty) or INSERT_SUBTREE (one leaf). + by_name = {s.name: s for s in statements} + removable = [ + i + for i, s in enumerate(statements) + if s.has_list + and s.name != "root" + and any(s.name in o.children for o in statements) + and all( + by_name.get(c) is not None and not by_name[c].has_list + for c in s.children + ) + ] + if not removable: + continue + idx = rng.choice(removable) + victim = statements[idx] + if victim.comp not in self.comp_index: + continue + if victim.rest not in CONTAINER_RESTS: + # The inverse (re-minting this subtree) is only expressible + # for canonically-rested containers; skip, never fake it. + continue + rest_idx = CONTAINER_RESTS.index(victim.rest) + parent_name = next( + o.name for o in statements if victim.name in o.children + ) + inverse: Edit | None = None + if not victim.children: + inverse = Edit( + ACTION_ADD_CONTAINER, 0, self.comp_index[victim.comp], + target=rest_idx, + ) + elif len(victim.children) == 1: + leaf = by_name[victim.children[0]] + slot = self._leaf_slot_index(leaf, inventory) + if leaf.comp in self.comp_index and slot is not None: + inverse = Edit( + ACTION_INSERT_SUBTREE, + 0, + self.comp_index[victim.comp], + slot, + target=rest_idx, + payload=self.comp_index[leaf.comp], + ) + if inverse is None: + continue + mutation = Edit(ACTION_REMOVE_CONTAINER, idx) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + parent_idx = next( + i for i, s in enumerate(mutated) if s.name == parent_name + ) + inverse = Edit( + inverse.action, parent_idx, inverse.comp, inverse.slot, + target=inverse.target, payload=inverse.payload, + ) + return mutated, inverse + if kind == ACTION_INSERT_SUBTREE: + # Mutation = transactional container+leaf subtree; inverse = + # REMOVE_CONTAINER of the minted root. + parents = [i for i, s in enumerate(statements) if s.has_list] + if not parents or not inventory or len(statements) + 2 > MAX_STMTS: + continue + parent_idx = rng.choice(parents) + root_comp = rng.choice( + [c for c in CONTAINER_COMPONENTS if c in self.comp_index] + ) + leaf_comp = rng.choice( + [c for c in LEAF_COMPONENTS if c in self.comp_index] + ) + slot = rng.randrange(min(len(inventory), MAX_SLOTS)) + mutation = Edit( + ACTION_INSERT_SUBTREE, + parent_idx, + self.comp_index[root_comp], + slot, + target=rng.randrange(len(CONTAINER_RESTS)), + payload=self.comp_index[leaf_comp], + ) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_REMOVE_CONTAINER, len(mutated) - 2) + return mutated, inverse + if kind == ACTION_REPLACE_SUBTREE: + # Mutation = replace the leaf of a canonical one-leaf container + # subtree; inverse = REPLACE_SUBTREE restoring old comp+slot. + by_name = {s.name: s for s in statements} + candidates = [ + i + for i, s in enumerate(statements) + if s.has_list + and len(s.children) == 1 + and by_name.get(s.children[0]) is not None + and not by_name[s.children[0]].has_list + ] + if not candidates or not inventory: + continue + idx = rng.choice(candidates) + leaf = by_name[statements[idx].children[0]] + old_slot = self._leaf_slot_index(leaf, inventory) + if leaf.comp not in self.comp_index or old_slot is None: + continue + choices = [ + c for c in LEAF_COMPONENTS if c in self.comp_index + ] + slot = rng.randrange(min(len(inventory), MAX_SLOTS)) + new_comp = rng.choice(choices) + if new_comp == leaf.comp and slot == old_slot: + continue + mutation = Edit( + ACTION_REPLACE_SUBTREE, idx, slot=slot, + payload=self.comp_index[new_comp], + ) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit( + ACTION_REPLACE_SUBTREE, idx, slot=old_slot, + payload=self.comp_index[leaf.comp], + ) + return mutated, inverse + if kind == ACTION_INSERT_STATEMENT: + # Mutation = canonical V0.5 statement; inverse = REMOVE it. + if len(statements) >= MAX_STMTS: + continue + payload = rng.randrange(len(V05_TEMPLATES)) + mutation = Edit(ACTION_INSERT_STATEMENT, payload=payload) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_REMOVE, len(mutated) - 1) + return mutated, inverse + if kind == ACTION_REPLACE_STATEMENT: + # Mutation = swap canonical V0.5 template; inverse = swap back. + candidates = [ + i for i, s in enumerate(statements) + if v05_template_index(s) is not None + ] + if not candidates: + continue + idx = rng.choice(candidates) + old_payload = v05_template_index(statements[idx]) + assert old_payload is not None + choices = [t for t in range(len(V05_TEMPLATES)) if t != old_payload] + if not choices: + continue + mutation = Edit( + ACTION_REPLACE_STATEMENT, idx, payload=rng.choice(choices) + ) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_REPLACE_STATEMENT, idx, payload=old_payload) + return mutated, inverse + if kind == ACTION_BIND_PLACEHOLDER: + # Mutation = rebind a leaf's slot; inverse = bind the old slot. + bindable = [ + i + for i, s in enumerate(statements) + if not s.has_list + and s.name != "root" + and s.comp in LEAF_COMPONENTS + ] + if not bindable or not inventory: + continue + idx = rng.choice(bindable) + old_slot = self._leaf_slot_index(statements[idx], inventory) + if old_slot is None or len(inventory) < 2: + continue + choices = [ + s for s in range(min(len(inventory), MAX_SLOTS)) if s != old_slot + ] + if not choices: + continue + mutation = Edit(ACTION_BIND_PLACEHOLDER, idx, slot=rng.choice(choices)) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_BIND_PLACEHOLDER, idx, slot=old_slot) + return mutated, inverse # Mutation = remove a leaf; inverse = ADD it back. removable = [ i @@ -348,6 +839,20 @@ def sample_mutation( return mutated, inverse return None + def _leaf_slot_index(self, stmt: Statement, inventory: list[str]) -> int | None: + """Inventory index of the placeholder bound by a leaf, else None.""" + body = stmt.rest.strip() + if not (body.startswith('"') or body.startswith("'")): + return None + try: + literal = json.loads(body) if body.startswith('"') else body[1:-1] + except Exception: # noqa: BLE001 + return None + if isinstance(literal, str) and literal in inventory: + index = inventory.index(literal) + return index if index < MAX_SLOTS else None + return None + @dataclass class TreeEditDiffusionConfig: @@ -423,7 +928,10 @@ def forward( class TreeEditDiffusionModel(nn.Module): """Prompt-conditioned Kapur-style edit policy + value search (X22).""" - CHECKPOINT_FORMAT = 1 + # Format 2 (SLM-305): action_head grew to N_ACTIONS=11 with the extended + # edit language. Format-1 checkpoints fail closed here; warm-start them + # via ``checkpoint_migrate.migrate_tree_edit_checkpoint``. + CHECKPOINT_FORMAT = 2 def __init__( self, @@ -592,7 +1100,8 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: comp_rows = [ i for i, e in enumerate(targets) - if e.action in {ACTION_REPLACE, ACTION_ADD} + if e.action + in {ACTION_REPLACE, ACTION_ADD, ACTION_ADD_CONTAINER, ACTION_INSERT_SUBTREE} ] if comp_rows: idx = torch.tensor(comp_rows, device=device) @@ -600,7 +1109,17 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: comp_loss = F.cross_entropy(out["comp"][idx], comp_t) loss = loss + comp_loss losses["comp"] = float(comp_loss.detach().cpu()) - slot_rows = [i for i, e in enumerate(targets) if e.action == ACTION_ADD] + slot_rows = [ + i + for i, e in enumerate(targets) + if e.action + in { + ACTION_ADD, + ACTION_INSERT_SUBTREE, + ACTION_REPLACE_SUBTREE, + ACTION_BIND_PLACEHOLDER, + } + ] if slot_rows: idx = torch.tensor(slot_rows, device=device) slot_t = torch.tensor( @@ -649,6 +1168,9 @@ def _enumerate_edits( (float(action_lp[ACTION_STOP]), Edit(ACTION_STOP)) ] n_comp = comp_lp.shape[0] + leaf_comps = [ + i for i, c in enumerate(self.space.components) if c in LEAF_COMPONENTS + ] for stmt in range(min(n_stmts, MAX_STMTS)): base = float(stmt_lp[stmt]) for comp in range(n_comp): @@ -658,19 +1180,78 @@ def _enumerate_edits( Edit(ACTION_REPLACE, stmt, comp), ) ) + for rest_idx in range(len(CONTAINER_RESTS)): + scored.append( + ( + float(action_lp[ACTION_ADD_CONTAINER]) + + base + + float(comp_lp[comp]), + Edit(ACTION_ADD_CONTAINER, stmt, comp, target=rest_idx), + ) + ) for slot in range(min(n_slots, MAX_SLOTS)): + slot_score = float(slot_lp[slot]) scored.append( ( float(action_lp[ACTION_ADD]) + base + float(comp_lp[comp]) - + float(slot_lp[slot]), + + slot_score, Edit(ACTION_ADD, stmt, comp, slot), ) ) + scored.append( + ( + float(action_lp[ACTION_BIND_PLACEHOLDER]) + + base + + slot_score, + Edit(ACTION_BIND_PLACEHOLDER, stmt, slot=slot), + ) + ) + for payload in leaf_comps: + for rest_idx in range(len(CONTAINER_RESTS)): + scored.append( + ( + float(action_lp[ACTION_INSERT_SUBTREE]) + + base + + float(comp_lp[comp]) + + slot_score, + Edit(ACTION_INSERT_SUBTREE, stmt, comp, slot, + target=rest_idx, payload=payload), + ) + ) + scored.append( + ( + float(action_lp[ACTION_REPLACE_SUBTREE]) + + base + + slot_score, + Edit(ACTION_REPLACE_SUBTREE, stmt, slot=slot, + payload=payload), + ) + ) scored.append( (float(action_lp[ACTION_REMOVE]) + base, Edit(ACTION_REMOVE, stmt)) ) + scored.append( + ( + float(action_lp[ACTION_REMOVE_CONTAINER]) + base, + Edit(ACTION_REMOVE_CONTAINER, stmt), + ) + ) + for payload in range(len(V05_TEMPLATES)): + scored.append( + ( + float(action_lp[ACTION_REPLACE_STATEMENT]) + base, + Edit(ACTION_REPLACE_STATEMENT, stmt, payload=payload), + ) + ) + for payload in range(len(V05_TEMPLATES)): + scored.append( + ( + float(action_lp[ACTION_INSERT_STATEMENT]), + Edit(ACTION_INSERT_STATEMENT, payload=payload), + ) + ) scored.sort(key=lambda pair: pair[0], reverse=True) return scored @@ -854,6 +1435,14 @@ def from_checkpoint( raise ValueError( f"checkpoint kind {payload.get('kind')!r} is not tree_edit_diffusion" ) + format_version = int(payload.get("format_version") or 1) + if format_version != cls.CHECKPOINT_FORMAT: + raise ValueError( + f"tree_edit_diffusion checkpoint format_version={format_version} " + f"is not supported (expected {cls.CHECKPOINT_FORMAT}); warm-start " + "older checkpoints via " + "slm_training.models.checkpoint_migrate.migrate_tree_edit_checkpoint" + ) tokenizer = OpenUITokenizer.load(path.with_suffix(".tokenizer.json")) config = TreeEditDiffusionConfig(**payload["config"]) model = cls(tokenizer, config=config, device=device) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 45fadec38..01d77c873 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -665,7 +665,7 @@ ] }, "evals.agentv": { - "version": "v2", + "version": "v3", "kind": "harness", "paths": [ "src/slm_training/evals/agentv.py", @@ -673,6 +673,11 @@ "tests/test_evals/test_agentv.py" ], "history": [ + { + "version": "v3", + "date": "2026-07-24", + "note": "Forward valid W3C run traces to the pinned AgentV runner for summary-only observational telemetry" + }, { "version": "v2", "date": "2026-07-24", @@ -696,6 +701,11 @@ "tests/test_scripts/test_run_cap2_operator_eval_fixture.py" ], "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: document opt-in LangSmith telemetry configuration in the shared README without changing the SLM-228 disposition" + }, { "version": "v1", "date": "2026-07-23", @@ -4540,7 +4550,7 @@ ] }, "harness.model_build.eval": { - "version": "v53", + "version": "v54", "kind": "harness", "paths": [ "src/slm_training/harnesses/model_build/debt_router.py", @@ -4551,6 +4561,11 @@ "scripts/evaluate_model.py" ], "history": [ + { + "version": "v54", + "date": "2026-07-24", + "note": "Add bounded opt-in LangSmith aggregate telemetry to the shared evaluation trace while keeping local evidence authoritative" + }, { "version": "v53", "date": "2026-07-24", @@ -6188,6 +6203,11 @@ "docs/design/iter-slm242-twotower-numeric-gates-20260721.md" ], "history": [ + { + "version": "v234", + "date": "2026-07-24", + "note": "no-bump: add tree-edit checkpoint format migration without changing TwoTower behavior" + }, { "version": "v234", "date": "2026-07-24", diff --git a/src/slm_training/runtime/telemetry/trace.py b/src/slm_training/runtime/telemetry/trace.py index 76ecce574..34f74ef07 100644 --- a/src/slm_training/runtime/telemetry/trace.py +++ b/src/slm_training/runtime/telemetry/trace.py @@ -9,6 +9,8 @@ import time import urllib.request import uuid +import warnings +from datetime import datetime, timezone from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -16,6 +18,157 @@ _CURRENT: contextvars.ContextVar["RunTrace | None"] = contextvars.ContextVar( "slm_run_trace", default=None ) +_ENV_PATH = Path(__file__).resolve().parents[4] / ".env" + + +def _enabled(name: str) -> bool: + return os.getenv(name, "").strip().lower() in {"1", "true", "yes", "on"} + + +def _key_value_env(name: str) -> dict[str, str]: + values: dict[str, str] = {} + for pair in os.getenv(name, "").split(","): + key, sep, value = pair.partition("=") + if sep and key.strip(): + values[key.strip()] = value.strip() + return values + + +def _timeout_seconds(name: str, default: float, maximum: float = 5.0) -> float: + try: + return min(maximum, max(0.1, float(os.getenv(name, default)))) + except ValueError: + return default + + +def _load_local_env() -> None: + """Load the repository's ignored local configuration without overriding CI.""" + try: + from dotenv import load_dotenv + except ImportError: + return + load_dotenv(_ENV_PATH, override=False) + + +def _langsmith_api_key() -> str | None: + value = os.getenv("LANGSMITH_API_KEY", "").strip() + if not value or value == "replace_with_rotated_langsmith_key": + return None + return value + + +def _langsmith_flush_seconds() -> float: + return _timeout_seconds("SLM_LANGSMITH_FLUSH_SECONDS", 2.0) + + +def _otlp_timeout_seconds() -> float: + """Convert the standard OTLP millisecond setting to a bounded timeout.""" + try: + value = float(os.getenv("OTEL_EXPORTER_OTLP_TIMEOUT", "1000")) / 1000 + except ValueError: + value = 1.0 + return min(5.0, max(0.1, value)) + + +class _LangSmithTrace: + """Best-effort summary exporter; local OTLP remains the source of truth.""" + + def __init__(self, trace: "RunTrace") -> None: + _load_local_env() + self.trace = trace + self.client: Any | None = None + self.error: str | None = None + api_key = _langsmith_api_key() + self.config = { + "enabled": bool(api_key) and _enabled("LANGSMITH_TRACING"), + "api_key_configured": bool(api_key), + "project": os.getenv("LANGSMITH_PROJECT", "slm-training"), + "endpoint": os.getenv("LANGSMITH_ENDPOINT") or None, + "workspace_id_configured": bool(os.getenv("LANGSMITH_WORKSPACE_ID")), + } + self.run_id = uuid.UUID(hex=trace.trace_id) + + def start(self) -> None: + if not self.config["enabled"]: + return + try: + from langsmith import Client + + self.client = Client( + api_key=_langsmith_api_key(), + api_url=self.config["endpoint"], + workspace_id=os.getenv("LANGSMITH_WORKSPACE_ID") or None, + omit_traced_runtime_info=True, + ) + self.client.create_run( + id=self.run_id, + project_name=self.config["project"], + name=f"slm.{self.trace.operation}", + run_type="chain", + inputs={"run_id": self.trace.run_id, "operation": self.trace.operation}, + start_time=datetime.fromtimestamp(self.trace.start_ns / 1e9, tz=timezone.utc), + extra={ + "metadata": { + "w3c_trace_id": self.trace.trace_id, + "service": "slm-training", + **self.trace.attributes, + } + }, + ) + except Exception as exc: # noqa: BLE001 - observability must never stop a run + self.client = None + self._failed(exc) + + def summary( + self, + name: str, + *, + inputs: dict[str, Any], + outputs: dict[str, Any], + metadata: dict[str, Any], + ) -> None: + if self.client is None: + return + try: + now = datetime.now(timezone.utc) + self.client.create_run( + id=uuid.uuid4(), + trace_id=self.run_id, + parent_run_id=self.run_id, + project_name=self.config["project"], + name=name, + run_type="tool", + inputs=inputs, + outputs=outputs, + start_time=now, + end_time=now, + extra={"metadata": {"w3c_trace_id": self.trace.trace_id, **metadata}}, + ) + except Exception as exc: # noqa: BLE001 - observability must never stop a run + self._failed(exc) + + def finish(self, error: BaseException | None) -> None: + if self.client is None: + return + try: + self.client.update_run( + self.run_id, + end_time=datetime.now(timezone.utc), + outputs={"status": "failed" if error else "completed"}, + error="run failed; inspect local trace" if error else None, + ) + self.client.flush(timeout=_langsmith_flush_seconds()) + except Exception as exc: # noqa: BLE001 - observability must never stop a run + self._failed(exc) + + def manifest(self) -> dict[str, Any]: + return {**self.config, "trace_id": self.trace.trace_id, "last_export_error": self.error} + + def _failed(self, exc: Exception) -> None: + self.error = type(exc).__name__ + warnings.warn( + f"LangSmith export failed: {self.error}", RuntimeWarning, stacklevel=3 + ) def _hex_id(size: int) -> str: @@ -53,17 +206,16 @@ def _endpoint(signal: str) -> str | None: base = specific or os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") or _first_peer() if not base: return None - return base if specific else f"{base.rstrip('/')}/v1/{signal}" + if specific: + return base + normalized = base.rstrip("/") + suffix = f"/v1/{signal}" + return normalized if normalized.endswith(suffix) else f"{normalized}{suffix}" def _headers() -> dict[str, str]: - raw = os.getenv("OTEL_EXPORTER_OTLP_HEADERS") - if raw: - headers: dict[str, str] = {} - for pair in raw.split(","): - key, sep, value = pair.partition("=") - if sep and key.strip(): - headers[key.strip()] = value.strip() + headers = _key_value_env("OTEL_EXPORTER_OTLP_HEADERS") + if headers: return headers token = os.getenv("SLM_OTEL_TOKEN") if not token and os.getenv("SLM_OTEL_AUTH", "").strip().lower() == "hf": @@ -100,6 +252,7 @@ def __post_init__(self) -> None: self.bundle = self.trace_root / self.trace_id self.instance_id = str(uuid.uuid4()) self._token = None + self._langsmith = _LangSmithTrace(self) self._write_manifest() if reference: reference.parent.mkdir(parents=True, exist_ok=True) @@ -125,6 +278,7 @@ def traceparent(self) -> str: def __enter__(self) -> "RunTrace": self._token = _CURRENT.set(self) + self._langsmith.start() self.log("run.started", attributes={"slm.operation": self.operation}) return self @@ -138,9 +292,22 @@ def __exit__(self, exc_type, exc, traceback) -> None: payload = self._trace_payload(time.time_ns(), status, str(exc) if exc else "") self._append("traces", payload) self._mirror("traces", payload) + self._langsmith.finish(exc) + self._write_manifest() if self._token is not None: _CURRENT.reset(self._token) + def record_summary( + self, + name: str, + *, + inputs: dict[str, Any], + outputs: dict[str, Any], + metadata: dict[str, Any], + ) -> None: + """Export a caller-curated aggregate only; never pass raw samples here.""" + self._langsmith.summary(name, inputs=inputs, outputs=outputs, metadata=metadata) + def domain_path(self, kind: str, name: str = "records.jsonl") -> Path: if not kind.replace("_", "").replace("-", "").isalnum(): raise ValueError(f"unsafe domain trace kind: {kind!r}") @@ -213,11 +380,17 @@ def _common_attributes(self) -> dict[str, Any]: return {"slm.run.id": self.run_id, "slm.operation": self.operation, **self.attributes} def _resource_attributes(self) -> dict[str, Any]: + configured = _key_value_env("OTEL_RESOURCE_ATTRIBUTES") return { - "service.name": "slm-training", - "service.namespace": "openui", - "service.version": "0.1.0", + "service.name": ( + os.getenv("OTEL_SERVICE_NAME") + or configured.pop("service.name", None) + or "slm-training" + ), + "service.namespace": configured.pop("service.namespace", "openui"), + "service.version": configured.pop("service.version", "0.1.0"), "service.instance.id": self.instance_id, + **configured, } def _append(self, signal: str, payload: dict[str, Any]) -> None: @@ -237,7 +410,8 @@ def _mirror(self, signal: str, payload: dict[str, Any]) -> None: method="POST", ) try: - with urllib.request.urlopen(request, timeout=2): # noqa: S310 + timeout = _otlp_timeout_seconds() + with urllib.request.urlopen(request, timeout=timeout): # noqa: S310 pass except OSError as exc: self._write_manifest(export_error=str(exc)) @@ -263,6 +437,7 @@ def _write_manifest(self, *, export_error: str | None = None) -> None: "otlp_json": True, "remote_endpoint_configured": bool(_endpoint("traces")), "last_export_error": export_error, + "langsmith": self._langsmith.manifest(), } path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") diff --git a/tests/test_evals/test_agentv.py b/tests/test_evals/test_agentv.py index 1bd9c341f..9269b6302 100644 --- a/tests/test_evals/test_agentv.py +++ b/tests/test_evals/test_agentv.py @@ -2,6 +2,7 @@ import json from pathlib import Path +from types import SimpleNamespace import slm_training.evals.agentv as agentv_module @@ -132,6 +133,39 @@ def test_agentv_contract_checks_fail_even_when_pass_flag_is_true(tmp_path) -> No assert published["criteria"]["failed"] == 1 +def test_agentv_forwards_w3c_trace_id_to_the_node_runner(tmp_path, monkeypatch) -> None: + runner = tmp_path / "runner.mjs" + runner.write_text("// fixture") + sdk_root = tmp_path / "sdk-root" + captured = {} + (tmp_path / "trace.json").write_text( + json.dumps({"trace_id": "0123456789abcdef0123456789abcdef"}) + ) + monkeypatch.setattr(agentv_module, "_agentv_runtime", lambda _: (runner, sdk_root)) + + def fake_run(command, **kwargs): + captured["command"] = command + return SimpleNamespace( + returncode=0, + stdout=json.dumps({"summary": {}, "artifacts": {}}), + stderr="", + ) + + monkeypatch.setattr(agentv_module.subprocess, "run", fake_run) + publish_agentv_evaluation( + tmp_path, + name="trace-link", + claim="fixture_wiring_not_ship", + cases=[{"id": "case", "criteria": "passes", "pass": True}], + ) + assert captured["command"][-4:] == [ + "--trace-id", + "0123456789abcdef0123456789abcdef", + "--run-id", + tmp_path.name, + ] + + def test_agentv_model_bundle_cannot_pass_a_smoke_only_run(tmp_path) -> None: published = publish_model_evaluation( tmp_path, diff --git a/tests/test_harnesses/experiments/test_slm299_edit_reachability.py b/tests/test_harnesses/experiments/test_slm299_edit_reachability.py index 639f5af03..9d9aba6c9 100644 --- a/tests/test_harnesses/experiments/test_slm299_edit_reachability.py +++ b/tests/test_harnesses/experiments/test_slm299_edit_reachability.py @@ -55,7 +55,9 @@ def test_reachable_leaf_replacement_from_seeded_state() -> None: ) assert case.verdict is Verdict.PROVEN_REACHABLE assert case.edit_lower_bound == 1 - assert case.path[0]["action"] == "REPLACE" + # Either the v1 REPLACE or the extended REPLACE_SUBTREE is a valid + # 1-edit proof here; BFS returns the first in deterministic order. + assert case.path[0]["action"] in {"REPLACE", "REPLACE_SUBTREE"} def test_seed_identity_is_zero_edits() -> None: @@ -71,12 +73,26 @@ def test_multi_container_target_needs_container_add() -> None: SEED, 'root = Stack([card], "column")\ntitle = TextContent(":t")\ncard = Card([title], "column")', slot_inventory=[":t"], + mode="v1", ) assert case.verdict is Verdict.PROVEN_UNREACHABLE assert case.reason_code == "needs_container_add" assert case.edit_lower_bound is None +def test_multi_container_target_reachable_in_extended_space() -> None: + # SLM-305: the real ADD_CONTAINER / BIND_PLACEHOLDER actions close the + # needs_container_add gap; the synthetic lane is retired for this space. + case = analyze_reachability( + SEED, + 'root = Stack([card], "column")\ntitle = TextContent(":t")\ncard = Card([title], "column")', + slot_inventory=[":t"], + ) + assert case.verdict is Verdict.PROVEN_REACHABLE + actions = [step["action"] for step in case.path] + assert "ADD_CONTAINER" in actions or "INSERT_SUBTREE" in actions + + # (c) state/query/action (V0.5) program forms ------------------------------ @pytest.mark.parametrize( @@ -108,17 +124,32 @@ def test_pack_feature_targets_are_unsupported(target: str) -> None: def test_tiny_budget_yields_unknown_not_unreachable() -> None: case = analyze_reachability( - SEED, _FIVE_LEAF_TARGET, slot_inventory=[":x"], max_edits=2 + SEED, _FIVE_LEAF_TARGET, slot_inventory=[":x"], max_edits=2, mode="v1" ) assert case.verdict is Verdict.UNKNOWN_BUDGET assert case.reason_code == "budget" # And it is genuinely reachable with enough budget — the UNKNOWN was a - # budget artifact, never a proof of unreachability. - full = analyze_reachability(SEED, _FIVE_LEAF_TARGET, slot_inventory=[":x"]) + # budget artifact, never a proof of unreachability (v1 space keeps the + # branching small enough to certify the depth-5 bound). + full = analyze_reachability( + SEED, _FIVE_LEAF_TARGET, slot_inventory=[":x"], mode="v1" + ) assert full.verdict is Verdict.PROVEN_REACHABLE assert full.edit_lower_bound == 5 +def test_extended_budget_stop_is_explicit_unknown_never_unreachable() -> None: + # The extended space's branching means deep targets exhaust the node + # budget: the verdict must be UNKNOWN_BUDGET, never a silent invariant or + # a false unreachability proof. + case = analyze_reachability( + SEED, _FIVE_LEAF_TARGET, slot_inventory=[":x"], node_budget=40 + ) + assert case.verdict is Verdict.UNKNOWN_BUDGET + assert case.reason_code == "budget" + assert case.details["stop"] == "node_budget" + + def test_unknowns_are_excluded_from_reachable_fraction() -> None: cases = [ ("hit", analyze_reachability(SEED, SEED, slot_inventory=[":x"])), @@ -142,7 +173,7 @@ def test_unknowns_are_excluded_from_reachable_fraction() -> None: def test_synthetic_add_container_flips_verdict_and_is_marked() -> None: - real = analyze_reachability(SEED, _CARD_TARGET, slot_inventory=[":x"]) + real = analyze_reachability(SEED, _CARD_TARGET, slot_inventory=[":x"], mode="v1") assert real.verdict is Verdict.PROVEN_UNREACHABLE assert real.reason_code == "needs_container_add" @@ -151,6 +182,7 @@ def test_synthetic_add_container_flips_verdict_and_is_marked() -> None: _CARD_TARGET, slot_inventory=[":x"], extra_actions=[add_container_action()], + mode="v1", ) assert what_if.verdict is Verdict.PROVEN_REACHABLE assert what_if.details["uses_synthetic_actions"] is True @@ -175,7 +207,7 @@ def _random_statements(rng: random.Random) -> list: def test_transition_parity_with_tree_edit_space() -> None: # The analyzer must enumerate exactly the children TreeEditSpace.apply - # admits — no more, no fewer — on random reachable states. + # admits — no more, no fewer — on random reachable states (v1 space). from slm_training.harnesses.experiments.slm299_edit_reachability import ( _enumerate_children, ) @@ -187,7 +219,7 @@ def test_transition_parity_with_tree_edit_space() -> None: statements = _random_statements(rng) via_analyzer = { render_statements(child) - for child, _ in _enumerate_children(space, statements, inventory) + for child, _ in _enumerate_children(space, statements, inventory, mode="v1") } via_reference: set[str] = set() n_comp = len(space.components) @@ -208,6 +240,47 @@ def test_transition_parity_with_tree_edit_space() -> None: assert via_analyzer == via_reference +def test_extended_enumeration_is_superset_and_deterministic() -> None: + # Extended mode enumerates every v1 child plus the new real actions, in a + # deterministic order (same state → same enumeration). + from slm_training.harnesses.experiments.slm299_edit_reachability import ( + _enumerate_children, + ) + + space = TreeEditSpace() + inventory = [":x", ":y"] + statements = parse_statements(SEED) + assert statements is not None + v1 = _enumerate_children(space, statements, inventory, mode="v1") + ext_first = _enumerate_children(space, statements, inventory, mode="extended") + ext_second = _enumerate_children(space, statements, inventory, mode="extended") + assert [a for _, a in ext_first] == [a for _, a in ext_second] + assert [ + render_statements(child) for child, _ in ext_first + ] == [render_statements(child) for child, _ in ext_second] + v1_keys = {render_statements(child) for child, _ in v1} + ext_keys = {render_statements(child) for child, _ in ext_first} + assert v1_keys <= ext_keys + ext_actions = {action["action"] for _, action in ext_first} + assert {"ADD_CONTAINER", "INSERT_STATEMENT", "INSERT_SUBTREE"} <= ext_actions + + +def test_v05_canonical_template_target_is_reachable_in_extended_space() -> None: + target = ( + 'root = Stack([], "column")\n' + 'q0 = Query("tool", {arg: $x}, {default: []}, 15)' + ) + extended = analyze_reachability(SEED, target, slot_inventory=[":x"]) + assert extended.verdict is Verdict.PROVEN_REACHABLE + assert any(step["action"] == "INSERT_STATEMENT" for step in extended.path) + v1 = analyze_reachability(SEED, target, slot_inventory=[":x"], mode="v1") + assert v1.verdict is Verdict.PROVEN_UNREACHABLE + # v1 has no statement actions at all: the pack line is either flagged as + # an unsupported pack feature (marker name prefix) or as a component the + # UI statement space cannot produce. + assert v1.reason_code in {"unsupported_pack_feature", "unsupported_component"} + + # (g) ship-gate integration ------------------------------------------------- def _gate_suites() -> dict[str, dict[str, object]]: @@ -286,10 +359,12 @@ def _mini_corpora() -> dict[str, list[dict[str, object]]]: def test_audit_report_is_deterministic() -> None: first = build_report( - _mini_corpora(), max_edits=8, node_budget=200, generated_at="2026-07-24T00:00:00Z" + _mini_corpora(), max_edits=8, node_budget=200, generated_at="2026-07-24T00:00:00Z", + mode="v1", ) second = build_report( - _mini_corpora(), max_edits=8, node_budget=200, generated_at="2026-07-24T00:00:00Z" + _mini_corpora(), max_edits=8, node_budget=200, generated_at="2026-07-24T00:00:00Z", + mode="v1", ) first.pop("version_stamp") second.pop("version_stamp") @@ -298,3 +373,18 @@ def test_audit_report_is_deterministic() -> None: assert first["suites"]["smoke"]["reachable_fraction"] == 0.5 assert first["suites"]["ood"]["status"] == "corpus_unavailable" assert first["x22_evidence_annotations"] + + +def test_audit_compare_reports_old_vs_extended() -> None: + payload = build_report( + _mini_corpora(), max_edits=8, node_budget=200, + generated_at="2026-07-24T00:00:00Z", mode="extended", compare=True, + ) + assert payload["suites"]["smoke"]["reachable_fraction"] == 1.0 + assert payload["suites_v1"]["smoke"]["reachable_fraction"] == 0.5 + comp = payload["old_vs_extended"]["smoke"] + assert comp["n_verdict_flips"] == 1 + assert comp["verdict_flips"][0]["id"] == "miss" + assert "version_stamp" in payload + md = render_markdown(payload) + assert "Old (v1) vs extended" in md diff --git a/tests/test_harnesses/model_build/test_eval_gates.py b/tests/test_harnesses/model_build/test_eval_gates.py index 37472d318..ce08fe2c7 100644 --- a/tests/test_harnesses/model_build/test_eval_gates.py +++ b/tests/test_harnesses/model_build/test_eval_gates.py @@ -23,6 +23,7 @@ from slm_training.harnesses.model_build.eval_runner import ( _effective_evaluation_policy, _is_meaningful_program, + _record_langsmith_evaluation, component_type_recall, evaluate, evaluate_suites, @@ -38,6 +39,44 @@ from slm_training.models.decode_stats import DecodeStats +def test_langsmith_evaluation_summary_excludes_per_example_details( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured = {} + + class Trace: + def record_summary(self, name, **kwargs) -> None: + captured["name"] = name + captured.update(kwargs) + + monkeypatch.setattr( + "slm_training.runtime.telemetry.current_trace", lambda: Trace() + ) + _record_langsmith_evaluation( + SimpleNamespace(run_id="safe-summary"), + suites={ + "smoke": { + "n": 1, + "parse_rate": 1.0, + "details": [{"prompt": "do not export", "completion": "secret"}], + } + }, + scoreboard={ + "run_class": "scratch_matrix", + "checkpoint_sha256": "a" * 64, + "eval_data_manifest_sha": "b" * 64, + "code_git_sha": "c" * 40, + "version_stamp": {"stamp_schema": "version_stamp/v1"}, + "gates": {"pass": False, "failures": ["smoke:parse_rate"]}, + "agentv": {"summary": {"passed": 0, "failed": 1}}, + }, + ) + assert captured["name"] == "evaluation.summary" + assert captured["outputs"]["suites"]["smoke"] == {"n": 1, "parse_rate": 1.0} + assert "details" not in str(captured) + assert "do not export" not in str(captured) + + def test_evaluation_policy_reports_loaded_checkpoint_settings() -> None: config = ModelBuildConfig( train_dir=Path("train"), diff --git a/tests/test_models/test_tree_edit_diffusion.py b/tests/test_models/test_tree_edit_diffusion.py index 1b1cfea4e..368bf729f 100644 --- a/tests/test_models/test_tree_edit_diffusion.py +++ b/tests/test_models/test_tree_edit_diffusion.py @@ -54,6 +54,166 @@ def test_mutations_preserve_validity_and_inverse_restores() -> None: assert restored_any +# --- SLM-305 extended edit language ----------------------------------------- + +from slm_training.models.tree_edit_diffusion import ( # noqa: E402 + ACTION_ADD_CONTAINER, + ACTION_BIND_PLACEHOLDER, + ACTION_INSERT_STATEMENT, + ACTION_INSERT_SUBTREE, + ACTION_REMOVE, + ACTION_REMOVE_CONTAINER, + ACTION_REPLACE_STATEMENT, + ACTION_REPLACE_SUBTREE, + Edit, +) + + +def test_render_is_byte_stable_for_fixture_programs() -> None: + statements = parse_statements(PROGRAM) + assert statements is not None + assert render_statements(statements) == PROGRAM + + +def test_extended_actions_apply_and_invert_exactly() -> None: + """Every new action applies only under its preconditions, stays valid, and + its supervised inverse restores the exact prior program text.""" + space = TreeEditSpace() + ci = space.comp_index + base = parse_statements(PROGRAM) + assert base is not None + + def apply(edit: Edit, stmts=None): + nxt = space.apply(stmts or base, edit, INVENTORY) + assert nxt is not None, edit + validate(render_statements(nxt)) + return nxt + + # ADD_CONTAINER <-> REMOVE_CONTAINER (empty subtree: exact safe inverse) + added = apply(Edit(ACTION_ADD_CONTAINER, 0, ci["Card"])) + assert len(added) == len(base) + 1 + assert render_statements( + apply(Edit(ACTION_REMOVE_CONTAINER, len(added) - 1), added) + ) == PROGRAM + # REMOVE_CONTAINER rejects non-empty nested subtrees (fail closed) + assert space.apply(base, Edit(ACTION_REMOVE_CONTAINER, 0), INVENTORY) is None + # INSERT_SUBTREE <-> REMOVE_CONTAINER (container + leaf child) + inserted = apply(Edit(ACTION_INSERT_SUBTREE, 0, ci["Stack"], 1, payload=ci["Button"])) + assert render_statements( + apply(Edit(ACTION_REMOVE_CONTAINER, len(inserted) - 2), inserted) + ) == PROGRAM + # REPLACE_SUBTREE swaps the single leaf of a canonical subtree, same root + replaced = apply(Edit(ACTION_REPLACE_SUBTREE, 1, slot=1, payload=ci["Image"])) + assert replaced[1].comp == "Card" # root kind preserved + assert render_statements( + apply(Edit(ACTION_REPLACE_SUBTREE, 1, slot=0, payload=ci["TextContent"]), replaced) + ) == PROGRAM + # INSERT_STATEMENT / REPLACE_STATEMENT (V0.5 canonical forms) + REMOVE + with_query = apply(Edit(ACTION_INSERT_STATEMENT, payload=0)) + assert with_query[-1].comp == "Query" + swapped = apply(Edit(ACTION_REPLACE_STATEMENT, len(with_query) - 1, payload=1), with_query) + assert swapped[-1].comp == "Mutation" + assert render_statements( + apply(Edit(ACTION_REPLACE_STATEMENT, len(swapped) - 1, payload=0), swapped) + ) == render_statements(with_query) + removed = space.apply(with_query, Edit(ACTION_REMOVE, len(with_query) - 1), INVENTORY) + assert removed is not None + assert render_statements(removed) == PROGRAM + # BIND_PLACEHOLDER rebinds a leaf slot; references (children lists) stay + bound = apply(Edit(ACTION_BIND_PLACEHOLDER, 4, slot=0)) + assert bound[1].children == base[1].children + assert render_statements( + apply(Edit(ACTION_BIND_PLACEHOLDER, 4, slot=1), bound) + ) == PROGRAM + # Out-of-precondition edits fail closed. + assert space.apply(base, Edit(ACTION_BIND_PLACEHOLDER, 0, slot=0), INVENTORY) is None + assert space.apply(base, Edit(ACTION_REPLACE_STATEMENT, 0, payload=0), INVENTORY) is None + + +def test_extended_sample_mutation_loop_restores() -> None: + space = TreeEditSpace() + rng = random.Random(11) + statements = parse_statements(PROGRAM) + assert statements is not None + seen_actions: set[int] = set() + for _ in range(80): + step = space.sample_mutation(statements, INVENTORY, rng) + if step is None: + continue + mutated, inverse = step + validate(render_statements(mutated)) + repaired = space.apply(mutated, inverse, INVENTORY) + assert repaired is not None + validate(render_statements(repaired)) + seen_actions.add(inverse.action) + # The seeded loop exercised the extended inverse-edit supervision. + assert seen_actions - {0} + + +def test_edit_new_fields_default() -> None: + old = Edit(1, 2, 3, 4) + assert old.target == 0 and old.payload == 0 + assert old == Edit(1, 2, 3, 4, 0, 0) + + +def test_checkpoint_format2_fail_closed_and_migration(tmp_path) -> None: + records = [ + ExampleRecord( + id="a", + prompt="Hero card with title, body, and a CTA button.", + openui=PROGRAM, + placeholders=INVENTORY, + ) + ] + cfg = TreeEditDiffusionConfig( + d_model=32, n_heads=4, context_layers=1, denoiser_layers=1, seed=3, + ) + model = TreeEditDiffusionModel.from_records(records, config=cfg, device="cpu") + path = tmp_path / "ckpt.pt" + model.save(path) + # Round-trip at format 2. + loaded = TreeEditDiffusionModel.from_checkpoint(path, device="cpu") + assert loaded.policy.action_head.out_features == model.policy.action_head.out_features + + # Simulate a format-1 checkpoint: shrink the action head to 4 rows. + import torch as _torch + + payload = _torch.load(path, map_location="cpu", weights_only=False) + payload["format_version"] = 1 + sd = payload["state_dict"] + old_w = sd["policy.action_head.weight"][:4].clone() + old_b = sd["policy.action_head.bias"][:4].clone() + sd["policy.action_head.weight"] = old_w + sd["policy.action_head.bias"] = old_b + old_path = tmp_path / "ckpt_v1.pt" + _torch.save(payload, old_path) + (tmp_path / "ckpt_v1.tokenizer.json").write_text( + (tmp_path / "ckpt.tokenizer.json").read_text(encoding="utf-8"), + encoding="utf-8", + ) + + # Unmigrated old checkpoint fails closed with a clear error. + with pytest.raises(ValueError, match="format_version=1"): + TreeEditDiffusionModel.from_checkpoint(old_path, device="cpu") + + # Migration warm-starts: old action rows preserved, new rows initialized. + from slm_training.models.checkpoint_migrate import migrate_tree_edit_checkpoint + + out_path = tmp_path / "ckpt_migrated.pt" + report = migrate_tree_edit_checkpoint( + source_checkpoint=old_path, output_checkpoint=out_path + ) + assert report["source_format_version"] == 1 + assert report["output_format_version"] == 2 + assert report["preserved_action_head_rows"] == 4 + assert (tmp_path / "ckpt_migrated.migrate.json").exists() + migrated = TreeEditDiffusionModel.from_checkpoint(out_path, device="cpu") + new_w = migrated.policy.action_head.weight + assert new_w.shape[0] == model.policy.action_head.weight.shape[0] + assert _torch.allclose(new_w[:4], old_w) + assert _torch.allclose(migrated.policy.action_head.bias[:4], old_b) + + def test_training_loss_decode_all_valid_and_checkpoint(tmp_path) -> None: records = [ ExampleRecord( diff --git a/tests/test_runtime_trace.py b/tests/test_runtime_trace.py index a793be0f2..6702bb21d 100644 --- a/tests/test_runtime_trace.py +++ b/tests/test_runtime_trace.py @@ -1,11 +1,23 @@ from __future__ import annotations import json +import sys +import uuid from pathlib import Path +from types import SimpleNamespace +import pytest + +import slm_training.runtime.telemetry.trace as trace_module from slm_training.runtime.telemetry import run_trace +@pytest.fixture(autouse=True) +def _disable_real_langsmith_exports(monkeypatch: pytest.MonkeyPatch) -> None: + """Focused tests use fakes when exercising the remote publisher.""" + monkeypatch.setenv("LANGSMITH_TRACING", "false") + + def test_run_trace_is_w3c_correlated_and_reused(tmp_path: Path) -> None: run_dir = tmp_path / "runs" / "demo" trace_root = tmp_path / "traces" @@ -37,6 +49,113 @@ def test_domain_trace_path_is_centralized(tmp_path: Path) -> None: assert path == tmp_path / trace.trace_id / "domain" / "synthesis" / "records.jsonl" +def test_langsmith_summary_uses_the_w3c_trace_and_safe_payload( + tmp_path: Path, monkeypatch +) -> None: + calls: list[tuple[str, tuple, dict]] = [] + + class FakeClient: + def __init__(self, **kwargs) -> None: + calls.append(("client", (), kwargs)) + + def create_run(self, **kwargs) -> None: + calls.append(("create", (), kwargs)) + + def update_run(self, *args, **kwargs) -> None: + calls.append(("update", args, kwargs)) + + def flush(self, **kwargs) -> None: + calls.append(("flush", (), kwargs)) + + monkeypatch.setitem(sys.modules, "langsmith", SimpleNamespace(Client=FakeClient)) + monkeypatch.setenv("LANGSMITH_API_KEY", "test-key") + monkeypatch.setenv("LANGSMITH_TRACING", "true") + run_dir = tmp_path / "runs" / "langsmith" + with run_trace("langsmith", "eval", run_dir=run_dir) as trace: + trace.record_summary( + "evaluation.summary", + inputs={"run_id": "langsmith", "suites": ["smoke"]}, + outputs={"suites": {"smoke": {"parse_rate": 1.0}}}, + metadata={"version_stamp": {"stamp_schema": "version_stamp/v1"}}, + ) + + creates = [payload for kind, _, payload in calls if kind == "create"] + assert len(creates) == 2 + root, child = creates + assert root["id"] == uuid.UUID(hex=trace.trace_id) + assert child["trace_id"] == root["id"] + assert child["parent_run_id"] == root["id"] + assert "prompt" not in json.dumps(child, default=str) + manifest = json.loads((tmp_path / "traces" / trace.trace_id / "manifest.json").read_text()) + assert manifest["langsmith"]["enabled"] is True + assert manifest["langsmith"]["project"] == "slm-training" + assert ("flush", (), {"timeout": 2.0}) in calls + + +def test_langsmith_loads_the_repository_env_file(tmp_path: Path, monkeypatch) -> None: + calls: list[dict] = [] + + class FakeClient: + def __init__(self, **kwargs) -> None: + calls.append(kwargs) + + def create_run(self, **kwargs) -> None: + pass + + def update_run(self, *args, **kwargs) -> None: + pass + + def flush(self, **kwargs) -> None: + pass + + env_path = tmp_path / ".env" + env_path.write_text( + "LANGSMITH_TRACING=true\n" + "LANGSMITH_API_KEY=test-key\n" + "LANGSMITH_PROJECT=env-project\n", + encoding="utf-8", + ) + for name in ("LANGSMITH_TRACING", "LANGSMITH_API_KEY", "LANGSMITH_PROJECT"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(trace_module, "_ENV_PATH", env_path) + monkeypatch.setitem(sys.modules, "langsmith", SimpleNamespace(Client=FakeClient)) + with run_trace("env", "eval", trace_root=tmp_path) as trace: + pass + assert calls[0]["api_key"] == "test-key" + manifest = json.loads((tmp_path / trace.trace_id / "manifest.json").read_text()) + assert manifest["langsmith"]["project"] == "env-project" + assert manifest["langsmith"]["api_key_configured"] is True + + +def test_langsmith_placeholder_is_inert(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "replace_with_rotated_langsmith_key") + monkeypatch.setenv("LANGSMITH_TRACING", "true") + with run_trace("placeholder", "eval", trace_root=tmp_path) as trace: + pass + manifest = json.loads((tmp_path / trace.trace_id / "manifest.json").read_text()) + assert manifest["langsmith"]["enabled"] is False + assert manifest["langsmith"]["api_key_configured"] is False + + +def test_langsmith_export_failures_do_not_stop_local_tracing(tmp_path: Path, monkeypatch) -> None: + class FailingClient: + def __init__(self, **kwargs) -> None: + pass + + def create_run(self, **kwargs) -> None: + raise OSError("offline") + + monkeypatch.setitem(sys.modules, "langsmith", SimpleNamespace(Client=FailingClient)) + monkeypatch.setenv("LANGSMITH_API_KEY", "test-key") + monkeypatch.setenv("LANGSMITH_TRACING", "true") + with pytest.warns(RuntimeWarning, match="LangSmith export failed"): + with run_trace("offline", "eval", trace_root=tmp_path) as trace: + trace.log("still.local") + assert (tmp_path / trace.trace_id / "signals" / "traces").is_dir() + manifest = json.loads((tmp_path / trace.trace_id / "manifest.json").read_text()) + assert manifest["langsmith"]["last_export_error"] == "OSError" + + def test_endpoint_precedence_includes_peer_fallback(monkeypatch) -> None: from slm_training.runtime.telemetry.trace import _endpoint @@ -58,6 +177,32 @@ def test_endpoint_precedence_includes_peer_fallback(monkeypatch) -> None: assert _endpoint("logs") == "http://exact/ingest" +def test_otlp_standard_endpoint_and_resource_config_are_honored( + tmp_path: Path, monkeypatch +) -> None: + from slm_training.runtime.telemetry.trace import _endpoint, _otlp_timeout_seconds + + monkeypatch.delenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318/v1/logs/") + assert _endpoint("logs") == "http://collector:4318/v1/logs" + + monkeypatch.setenv("OTEL_SERVICE_NAME", "slm-training-ci") + monkeypatch.setenv( + "OTEL_RESOURCE_ATTRIBUTES", + "service.namespace=ci,deployment.environment=test", + ) + with run_trace("resources", "eval", trace_root=tmp_path) as trace: + attributes = trace._resource_attributes() + assert attributes["service.name"] == "slm-training-ci" + assert attributes["service.namespace"] == "ci" + assert attributes["deployment.environment"] == "test" + + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "1000") + assert _otlp_timeout_seconds() == 1.0 + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "90000") + assert _otlp_timeout_seconds() == 5.0 + + def test_mirror_headers_resolution(monkeypatch) -> None: from slm_training.runtime.telemetry.trace import _headers From 6dd710a7d13753052e3c44cef8ae3d311f05e467 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:51:20 -0500 Subject: [PATCH 04/10] SLM-305: extend tree edit language --- src/slm_training/resources/versions.json | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 01d77c873..0397d95b9 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -4475,7 +4475,7 @@ ] }, "harness.experiments.slm299_edit_reachability": { - "version": "v1", + "version": "v2", "kind": "harness", "paths": [ "src/slm_training/harnesses/experiments/slm299_edit_reachability.py", @@ -4483,9 +4483,17 @@ "tests/test_harnesses/experiments/test_slm299_edit_reachability.py", "docs/design/iter-slm299-edit-reachability-20260724.json", "docs/design/iter-slm299-edit-reachability-20260724.md", - "src/slm_training/models/tree_edit_diffusion.py" + "src/slm_training/models/tree_edit_diffusion.py", + "docs/design/iter-slm305-edit-language-20260724.json", + "docs/design/iter-slm305-edit-language-20260724.md", + "tests/test_models/test_tree_edit_diffusion.py" ], "history": [ + { + "version": "v2", + "date": "2026-07-24", + "note": "SLM-305: extend the valid-state edit language (ADD_CONTAINER/REMOVE_CONTAINER, INSERT/REPLACE_SUBTREE, V0.5 statement edits, BIND_PLACEHOLDER) with inverses, parser-backed validation, checkpoint format 2 + migration, and extended reachability mode" + }, { "version": "v1", "date": "2026-07-24", From ae5448c594233b304ea22720604a962ad041fe63 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:44:33 -0500 Subject: [PATCH 05/10] SLM-308: exact bounded distance value labels + progress calibration - slm308_distance_oracle.py: bounded reverse-BFS over canonical AST fingerprints via the real extended 11-action transitions; EXACT / BOUNDED / UNKNOWN labels (budget never conflated with distance); cache keyed by action-schema version, grammar sha, inventory, target and state hashes. - tree_edit_diffusion: value_label_mode (bounded_distance default for new configs; pre-field checkpoints inject mutation_count for behavior parity, format stays 2); normalized oracle cost-to-go with UNKNOWN masked out of MSE; pairwise progress margin loss (independently tested); decode paths never read gold distance (audit test). - Matched fixture experiment (mutation_count vs bounded_distance, identical budgets, near-gold + seed-trajectory states): rank corr 0.433->0.505 (+0.072 < 0.10), beam regret 0.889->0.889 (+0.0 < 0.05), Brier 0.031->0.004, UNKNOWN coverage 0.175. Preregistered thresholds written before results; verdict honestly rejected at this budget. - 16 new tests; slm308 component v1; slm299 component v3. --- .../iter-slm308-distance-value-20260724.json | 150 +++++ .../iter-slm308-distance-value-20260724.md | 33 + package-lock.json | 110 --- package.json | 4 - scripts/run_agentv_eval.mjs | 48 +- scripts/run_slm308_distance_value.py | 627 ++++++++++++++++++ .../experiments/slm308_distance_oracle.py | 398 +++++++++++ .../models/tree_edit_diffusion.py | 144 +++- src/slm_training/resources/versions.json | 32 +- .../experiments/test_slm308_distance_value.py | 357 ++++++++++ 10 files changed, 1757 insertions(+), 146 deletions(-) create mode 100644 docs/design/iter-slm308-distance-value-20260724.json create mode 100644 docs/design/iter-slm308-distance-value-20260724.md create mode 100644 scripts/run_slm308_distance_value.py create mode 100644 src/slm_training/harnesses/experiments/slm308_distance_oracle.py create mode 100644 tests/test_harnesses/experiments/test_slm308_distance_value.py diff --git a/docs/design/iter-slm308-distance-value-20260724.json b/docs/design/iter-slm308-distance-value-20260724.json new file mode 100644 index 000000000..c0ca56e41 --- /dev/null +++ b/docs/design/iter-slm308-distance-value-20260724.json @@ -0,0 +1,150 @@ +{ + "experiment": "slm308-distance-value", + "issue": "SLM-308", + "question": "Do bounded-distance oracle value labels beat mutation-count value labels at equal fixture budget (rank correlation + beam regret)?", + "preregistered_thresholds": { + "beam_regret_improvement_min": 0.05, + "rank_correlation_improvement_min": 0.1, + "direction": "arm_b(bounded_distance) minus arm_a(mutation_count)", + "verdict_rule": "adopted iff both thresholds are met, else rejected" + }, + "config": { + "steps": 8, + "batch_size": 4, + "seed": 0, + "n_records": 8, + "eval_max_depth": 8, + "eval_node_budget": 8 + }, + "oracle_cost": { + "n_states": 40, + "n_unknown": 7, + "unknown_coverage": 0.175, + "n_measurable": 17, + "wall_ms_total": 26468.863566005894, + "ms_per_state": 661.7215891501473, + "cache": { + "label_hits": 0, + "label_misses": 40, + "map_builds": 8, + "target_maps": 8, + "labels": 40, + "map_nodes": 64 + } + }, + "arms": { + "arm_a": { + "value_label_mode": "mutation_count", + "metrics": { + "n_measurable": 17, + "rank_correlation": 0.4330127018922193, + "concordance": 0.75, + "brier": 0.030793564056903686, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 8, + "mean_predicted_value": 0.6458271443843842, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 9, + "mean_predicted_value": 0.6430932150946723, + "mean_target": 0.75 + } + ], + "beam_regret_mean": 0.8888888888888888, + "beam_regret_n": 9, + "beam_regret_excluded": 8, + "by_origin": { + "near_gold": { + "n": 14, + "rank_correlation": 0.35805743701971643 + }, + "seed_trajectory": { + "n": 3, + "rank_correlation": null + } + } + }, + "training_metrics": { + "action": 1.8480353355407715, + "stmt": 2.9644241333007812, + "slot": 0.3795056641101837, + "value": 0.08967626839876175, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + }, + "arm_b": { + "value_label_mode": "bounded_distance", + "metrics": { + "n_measurable": 17, + "rank_correlation": 0.5051814855409226, + "concordance": 0.7916666666666666, + "brier": 0.004281158882868815, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 8, + "mean_predicted_value": 0.7873154133558273, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 9, + "mean_predicted_value": 0.7852203448613485, + "mean_target": 0.75 + } + ], + "beam_regret_mean": 0.8888888888888888, + "beam_regret_n": 9, + "beam_regret_excluded": 8, + "by_origin": { + "near_gold": { + "n": 14, + "rank_correlation": 0.4296689244236597 + }, + "seed_trajectory": { + "n": 3, + "rank_correlation": null + } + } + }, + "training_metrics": { + "action": 1.8418681621551514, + "stmt": 2.97049617767334, + "slot": 0.37653154134750366, + "value": 0.031629983335733414, + "pairwise_progress": 0.10255739837884903, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + }, + "improvements": { + "rank_correlation": 0.07216878364870327, + "concordance": 0.04166666666666663, + "beam_regret": 0.0, + "brier": 0.026512405174034873 + }, + "threshold_checks": { + "rank_correlation_ok": false, + "beam_regret_ok": false + }, + "verdict": "rejected", + "honesty": "Fixture-scale matched arms; UNKNOWN/unbounded states are excluded from distance-referenced metrics and counted, never coerced. A fixture verdict is wiring/label evidence, not a production ship claim.", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "6dd710a7d13753052e3c44cef8ae3d311f05e467", + "code_dirty": true, + "components": { + "harness.experiments.slm308_distance_value": "v1", + "harness.experiments.slm299_edit_reachability": "v3" + }, + "stamped_at": "2026-07-24T21:40:05.585150+00:00" + } +} diff --git a/docs/design/iter-slm308-distance-value-20260724.md b/docs/design/iter-slm308-distance-value-20260724.md new file mode 100644 index 000000000..20ca3e747 --- /dev/null +++ b/docs/design/iter-slm308-distance-value-20260724.md @@ -0,0 +1,33 @@ +# SLM-308 (LAR2-02): bounded-distance value labels vs mutation-count + +**Verdict: `rejected`** (fixture-scale matched arms; not a ship claim) + +## Preregistered thresholds (locked before results) + +- beam regret improvement (A−B) >= 0.05 +- rank correlation improvement (B−A) >= 0.1 +- rule: adopted iff both thresholds are met, else rejected + +## Headline + +| metric | arm A (mutation_count) | arm B (bounded_distance) | improvement | +| --- | --- | --- | --- | +| rank correlation | 0.4330 | 0.5052 | 0.0722 | +| concordance | 0.7500 | 0.7917 | 0.0417 | +| Brier | 0.0308 | 0.0043 | 0.0265 | +| beam regret (mean) | 0.8889 | 0.8889 | 0.0000 | + +## Coverage + oracle cost + +- UNKNOWN coverage 0.175 (7/40), measurable 17, oracle 661.7 ms/state, 64 nodes expanded + +## Calibration (arm B, distance bins d=0..8+) + +| d | n | mean predicted value | mean target | +| --- | --- | --- | --- | +| 1 | 8 | 0.787 | 0.875 | +| 2 | 9 | 0.785 | 0.750 | + +## Honesty + +Fixture-scale matched arms; UNKNOWN/unbounded states are excluded from distance-referenced metrics and counted, never coerced. A fixture verdict is wiring/label evidence, not a production ship claim. diff --git a/package-lock.json b/package-lock.json index c16585b33..31e6abda1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,10 +7,6 @@ "": { "name": "slm-training-web", "version": "0.1.0", - "dependencies": { - "langsmith": "^0.8.7", - "openai": "^6.49.0" - }, "devDependencies": { "@agentv/core": "4.42.4", "@playwright/mcp": "0.0.78", @@ -2700,12 +2696,6 @@ "node": ">= 0.6" } }, - "node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "license": "MIT" - }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -3377,39 +3367,6 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/langsmith": { - "version": "0.8.7", - "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.8.7.tgz", - "integrity": "sha512-QWcc7JwmGy+sPRJwqCf9xLiiSIhwc/i7oVwjQezkx6UzyFlokME3Wxy80qKqirTyJvWG1S9ylqtxBDpFRpas5g==", - "license": "MIT", - "dependencies": { - "p-queue": "6.6.2" - }, - "peerDependencies": { - "@opentelemetry/api": "*", - "@opentelemetry/exporter-trace-otlp-proto": "*", - "@opentelemetry/sdk-trace-base": "*", - "openai": "*", - "ws": ">=7" - }, - "peerDependenciesMeta": { - "@opentelemetry/api": { - "optional": true - }, - "@opentelemetry/exporter-trace-otlp-proto": { - "optional": true - }, - "@opentelemetry/sdk-trace-base": { - "optional": true - }, - "openai": { - "optional": true - }, - "ws": { - "optional": true - } - } - }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -3614,61 +3571,6 @@ "wrappy": "1" } }, - "node_modules/openai": { - "version": "6.49.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-6.49.0.tgz", - "integrity": "sha512-aYCc0C6L864eR6WSYIwQGyXriw/nIyZx0ObvhzOEVuk0zoBDpynjSbrionWI7q65B5H8jJX0DXR9snEzM6bfPg==", - "license": "Apache-2.0", - "peerDependencies": { - "@aws-sdk/credential-provider-node": ">=3.972.0 <4", - "@smithy/hash-node": ">=4.3.0 <5", - "@smithy/signature-v4": ">=5.4.0 <6", - "ws": "^8.18.0", - "zod": "^3.25 || ^4.0" - }, - "peerDependenciesMeta": { - "@aws-sdk/credential-provider-node": { - "optional": true - }, - "@smithy/hash-node": { - "optional": true - }, - "@smithy/signature-v4": { - "optional": true - }, - "ws": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, - "node_modules/p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/p-queue": { - "version": "6.6.2", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", - "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", - "license": "MIT", - "dependencies": { - "eventemitter3": "^4.0.4", - "p-timeout": "^3.2.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/p-retry": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", @@ -3683,18 +3585,6 @@ "node": ">=8" } }, - "node_modules/p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "license": "MIT", - "dependencies": { - "p-finally": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", diff --git a/package.json b/package.json index 1d4c0d837..05e70d356 100644 --- a/package.json +++ b/package.json @@ -27,9 +27,5 @@ }, "engines": { "node": ">=20 <23" - }, - "dependencies": { - "langsmith": "^0.8.7", - "openai": "^6.49.0" } } diff --git a/scripts/run_agentv_eval.mjs b/scripts/run_agentv_eval.mjs index 5a0d73657..7e18dd2b3 100644 --- a/scripts/run_agentv_eval.mjs +++ b/scripts/run_agentv_eval.mjs @@ -31,31 +31,33 @@ async function publishLangSmithSummary({ traceId, runId, experiment, result }) { const parentRunId = traceUuid(traceId); if (!langsmithEnabled() || !parentRunId) return; try { - const { Client } = await import("langsmith"); - const client = new Client({ - apiKey: process.env.LANGSMITH_API_KEY, - apiUrl: process.env.LANGSMITH_ENDPOINT, - workspaceId: process.env.LANGSMITH_WORKSPACE_ID, - omitTracedRuntimeInfo: true, - }); const now = new Date().toISOString(); - await client.createRun({ - id: randomUUID(), - trace_id: parentRunId, - parent_run_id: parentRunId, - project_name: process.env.LANGSMITH_PROJECT || "slm-training", - name: "agentv.publication", - run_type: "tool", - inputs: { run_id: runId, experiment }, - outputs: { summary: result.summary }, - start_time: now, - end_time: now, - extra: { metadata: { w3c_trace_id: traceId, sdk: "@agentv/core" } }, + const endpoint = `${(process.env.LANGSMITH_ENDPOINT || "https://api.smith.langchain.com").replace(/\/$/, "")}/runs`; + const response = await fetch(endpoint, { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": process.env.LANGSMITH_API_KEY, + ...(process.env.LANGSMITH_WORKSPACE_ID + ? { "x-tenant-id": process.env.LANGSMITH_WORKSPACE_ID } + : {}), + }, + body: JSON.stringify({ + id: randomUUID(), + trace_id: parentRunId, + parent_run_id: parentRunId, + project_name: process.env.LANGSMITH_PROJECT || "slm-training", + name: "agentv.publication", + run_type: "tool", + inputs: { run_id: runId, experiment }, + outputs: { summary: result.summary }, + start_time: now, + end_time: now, + extra: { metadata: { w3c_trace_id: traceId, sdk: "@agentv/core" } }, + }), + signal: AbortSignal.timeout(500), }); - await Promise.race([ - client.flush(), - new Promise((resolve) => setTimeout(resolve, 500)), - ]); + if (!response.ok) throw new Error(`HTTP ${response.status}`); } catch (error) { console.warn(`LangSmith AgentV export failed: ${String(error)}`); } diff --git a/scripts/run_slm308_distance_value.py b/scripts/run_slm308_distance_value.py new file mode 100644 index 000000000..e02c7a500 --- /dev/null +++ b/scripts/run_slm308_distance_value.py @@ -0,0 +1,627 @@ +#!/usr/bin/env python3 +"""SLM-308 (LAR2-02): matched fixture experiment — mutation-count vs bounded-distance value labels. + +Question: does replacing the X22 tree-edit model's mutation-count value +labels (``1 - applied/(max_chain+1)``) with SLM-308 oracle labels (normalized +cost-to-go from the bounded reverse-BFS distance oracle + pairwise +parent/improving-child ranking) improve value quality and value-guided beam +selection, at equal model / data / optimizer budget? + +Preregistered (written into the output payload BEFORE any result): + +- primary: beam selection regret vs oracle-best must improve by **>= 0.05** + absolute (arm A regret − arm B regret, distance units); +- secondary: Spearman rank correlation between value and oracle distance + must improve by **>= 0.10** (arm B − arm A); +- verdict ``adopted`` iff both thresholds are met, else ``rejected`` — + honestly computed from the measured arms, never narrated. + +Arms share the tiny fixture corpus, model init seed, mutation chains, batch +order, and optimizer budget; they differ ONLY in ``value_label_mode``. +Evaluation covers near-gold states (gold mutated 1..3 edits) AND +seed-trajectory states (random walks from the decode seed): Spearman rank +correlation, pairwise concordance, Brier + per-bin calibration over distance +bins d=0..8+, beam selection regret vs oracle-best, UNKNOWN coverage, and +oracle cost (nodes expanded / ms). States the oracle cannot measure (UNKNOWN +or unbounded) are excluded from distance-referenced metrics and counted, +never coerced. + +Writes ``docs/design/iter-slm308-distance-value-20260724.{json,md}``. + +Example: + python -m scripts.run_slm308_distance_value --steps 8 +""" + +from __future__ import annotations + +import argparse +import json +import random +import time +from pathlib import Path + +import torch + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm308_distance_oracle import ( + DistanceKind, + cache_stats, + clear_caches, + distance_to_target, + effective_distance, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_STOP, + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + TreeEditSpace, + parse_statements, + render_statements, +) +from slm_training.versioning import build_version_stamp + +EXPERIMENT_ID = "slm308-distance-value" +DEFAULT_JSON_OUT = Path("docs/design/iter-slm308-distance-value-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm308-distance-value-20260724.md") + +# PREREGISTERED improvement thresholds (locked before any run; do not edit +# after outcomes are visible — deviations are append-only and exploratory). +PREREGISTERED_THRESHOLDS = { + "beam_regret_improvement_min": 0.05, + "rank_correlation_improvement_min": 0.10, + "direction": "arm_b(bounded_distance) minus arm_a(mutation_count)", + "verdict_rule": "adopted iff both thresholds are met, else rejected", +} + +EVAL_MAX_DEPTH = 8 +EVAL_NODE_BUDGET = 8 + +# Tiny fixture corpus: small valid programs with 1-2 placeholders. +FIXTURE_PROGRAMS: list[tuple[str, str, list[str]]] = [ + ("hero with cta", 'root = Stack([t, b], "column")\nt = TextContent(":hero.title")\nb = Button(":cta.label")', [":hero.title", ":cta.label"]), + ("simple text", 'root = Stack([t], "column")\nt = TextContent(":body")', [":body"]), + ("form with input", 'root = Form([i, s], "column")\ni = TextInput(":form.email")\ns = Button(":form.submit")', [":form.email", ":form.submit"]), + ("card with image", 'root = Stack([c], "column")\nc = Card([im, t], "column")\nim = Image(":img.src")\nt = TextContent(":img.caption")', [":img.src", ":img.caption"]), + ("two texts", 'root = Stack([a, b], "column")\na = TextContent(":title")\nb = TextContent(":subtitle")', [":title", ":subtitle"]), + ("button only", 'root = Stack([b], "column")\nb = Button(":cta")', [":cta"]), + ("nested card", 'root = Stack([c], "column")\nc = Card([t], "column")\nt = TextContent(":card.body")', [":card.body"]), + ("form pair", 'root = Form([i], "column")\ni = TextInput(":q")', [":q"]), +] + +SEED_SOURCE = 'root = Stack([], "column")' + + +def build_fixture_records() -> list[ExampleRecord]: + return [ + ExampleRecord( + id=f"fixture_{i:02d}", + prompt=prompt, + openui=openui, + placeholders=list(placeholders), + split="train", + ) + for i, (prompt, openui, placeholders) in enumerate(FIXTURE_PROGRAMS) + ] + + +# --- metrics ----------------------------------------------------------------- + + +def spearman(xs: list[float], ys: list[float]) -> float | None: + """Spearman rank correlation; None when undefined (ties everywhere).""" + + def ranks(values: list[float]) -> list[float]: + order = sorted(range(len(values)), key=lambda i: values[i]) + out = [0.0] * len(values) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and values[order[j + 1]] == values[order[i]]: + j += 1 + mean_rank = (i + j) / 2.0 + 1.0 + for k in range(i, j + 1): + out[order[k]] = mean_rank + i = j + 1 + return out + + if len(xs) < 3: + return None + rx, ry = ranks(xs), ranks(ys) + n = len(rx) + mx, my = sum(rx) / n, sum(ry) / n + cov = sum((a - mx) * (b - my) for a, b in zip(rx, ry)) + vx = sum((a - mx) ** 2 for a in rx) + vy = sum((b - my) ** 2 for b in ry) + if vx == 0 or vy == 0: + return None + return cov / (vx * vy) ** 0.5 + + +def concordance(preds: list[float], refs: list[float]) -> float | None: + """Fraction of comparable pairs the prediction orders correctly (ties in + reference skipped; prediction ties count as half).""" + total = 0 + correct = 0.0 + for i in range(len(preds)): + for j in range(i + 1, len(preds)): + if refs[i] == refs[j]: + continue + total += 1 + want = refs[i] < refs[j] + if preds[i] == preds[j]: + correct += 0.5 + elif (preds[i] < preds[j]) == want: + correct += 1.0 + return (correct / total) if total else None + + +def calibration_bins( + preds: list[float], refs: list[float], max_depth: int = EVAL_MAX_DEPTH +) -> list[dict[str, float | int]]: + """Per distance bin d=0..8+: mean predicted value vs mean normalized + cost-to-go target (1 - d/max_depth). Bins are by rounded reference + distance; d>=8 folds into the final bin.""" + bins: dict[int, list[tuple[float, float]]] = {} + for pred, ref in zip(preds, refs): + d = min(int(round(ref)), max_depth) + bins.setdefault(d, []).append((pred, 1.0 - min(ref, max_depth) / max_depth)) + return [ + { + "distance_bin": d, + "n": len(rows), + "mean_predicted_value": sum(r[0] for r in rows) / len(rows), + "mean_target": sum(r[1] for r in rows) / len(rows), + } + for d, rows in sorted(bins.items()) + ] + + +# --- eval states --------------------------------------------------------------- + + +def build_eval_states( + records: list[ExampleRecord], space: TreeEditSpace, *, seed: int = 20260724 +) -> list[dict]: + """Deterministic near-gold and seed-trajectory states per record.""" + states: list[dict] = [] + for idx, record in enumerate(records): + gold = parse_statements(record.openui) + assert gold is not None + inventory = [ + p if p.startswith(":") else f":{p}" for p in record.placeholders + ] + # Near-gold: mutate gold 1..3 edits. + rng = random.Random(seed + idx) + for k in (1, 2, 3): + current = gold + applied = 0 + for _ in range(k): + step = space.sample_mutation(current, inventory, rng) + if step is None: + break + current, _inverse = step + applied += 1 + if applied == 0: + continue + states.append( + { + "record_idx": idx, + "origin": "near_gold", + "witness": applied, + "statements": current, + "target": gold, + "inventory": inventory, + } + ) + # Seed-trajectory: random walks from the decode seed (1, 3 edits). + seed_statements = parse_statements(SEED_SOURCE) + assert seed_statements is not None + for k in (1, 3): + current = seed_statements + applied = 0 + for _ in range(k): + step = space.sample_mutation(current, inventory, rng) + if step is None: + break + current, _inverse = step + applied += 1 + if applied == 0: + continue + states.append( + { + "record_idx": idx, + "origin": "seed_trajectory", + "witness": None, # not a gold mutation chain: no witness + "statements": current, + "target": gold, + "inventory": inventory, + } + ) + return states + + +def label_eval_states(states: list[dict], space: TreeEditSpace) -> dict: + """Oracle labels for every eval state (shared across arms) + oracle cost.""" + clear_caches() + start = time.perf_counter() + n_unknown = 0 + for entry in states: + label = distance_to_target( + entry["statements"], + entry["target"], + space=space, + inventory=entry["inventory"], + max_depth=EVAL_MAX_DEPTH, + node_budget=EVAL_NODE_BUDGET, + upper_bound_witness=entry["witness"], + ) + entry["label"] = label + entry["ref_distance"] = effective_distance(label) + if label.kind is DistanceKind.UNKNOWN: + n_unknown += 1 + elapsed_ms = (time.perf_counter() - start) * 1000.0 + stats = cache_stats() + return { + "n_states": len(states), + "n_unknown": n_unknown, + "unknown_coverage": n_unknown / max(len(states), 1), + "n_measurable": sum(1 for e in states if e["ref_distance"] is not None), + "wall_ms_total": elapsed_ms, + "ms_per_state": elapsed_ms / max(len(states), 1), + "cache": stats, + } + + +# --- training ------------------------------------------------------------------ + + +def train_arm( + records: list[ExampleRecord], + mode: str, + *, + steps: int, + batch_size: int, + seed: int, +) -> TreeEditDiffusionModel: + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + seed=seed, + value_label_mode=mode, + context_backend="scratch", + max_chain=3, + ) + model = TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=3e-3) + order = random.Random(0) # identical batch order across arms + model.train() + for _ in range(steps): + batch = [records[order.randrange(len(records))] for _ in range(batch_size)] + loss = model.training_loss(batch) + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +# --- arm scoring ----------------------------------------------------------------- + + +@torch.no_grad() +def _model_value(model: TreeEditDiffusionModel, prompt: str, source: str) -> float: + ctx, ctx_pad = model._encode_context([prompt]) + out = model.policy( + model._state_batch([source]), model.tokenizer.pad_id, ctx, ctx_pad + ) + return float(out["value"][0]) + + +@torch.no_grad() +def beam_regret( + model: TreeEditDiffusionModel, + record: ExampleRecord, + entry: dict, +) -> float | None: + """Value-guided single-step selection regret vs oracle-best, in distance + units. Candidates mirror decode: STOP plus the policy's top applicable + edits; the value head picks; the oracle scores every candidate. None when + any candidate is oracle-unmeasurable (excluded, never coerced).""" + inventory = entry["inventory"] + statements = entry["statements"] + prompt = model._format_context( + record.prompt, design_md=record.design_md, slot_contract=inventory + ) + ctx, ctx_pad = model._encode_context([prompt]) + out = model.policy( + model._state_batch([render_statements(statements)]), + model.tokenizer.pad_id, + ctx, + ctx_pad, + ) + candidates = model._enumerate_edits(out, 0, len(statements), len(inventory)) + # (child statements or None for STOP) + options: list[list | None] = [] + expanded = 0 + for _score, edit in candidates: + if expanded >= model.config.expand_per_state: + break + if edit.action == ACTION_STOP: + options.append(None) + expanded += 1 + continue + child = model.space.apply(statements, edit, inventory) + if child is None: + continue + options.append(child) + expanded += 1 + if not options: + return None + child_sources = [ + render_statements(child) if child is not None else render_statements(statements) + for child in options + ] + values = model.policy( + model._state_batch(child_sources), + model.tokenizer.pad_id, + ctx.expand(len(options), -1, -1), + ctx_pad.expand(len(options), -1), + )["value"] + distances: list[float] = [] + for child in options: + # A candidate child of a witness-backed state is one edit further from + # gold along the same witness path: hi = witness + 1 is proven. + witness = ( + entry["witness"] + 1 if entry["witness"] is not None else None + ) + label = distance_to_target( + child if child is not None else statements, + entry["target"], + space=model.space, + inventory=inventory, + max_depth=EVAL_MAX_DEPTH, + node_budget=EVAL_NODE_BUDGET, + upper_bound_witness=witness, + ) + ref = effective_distance(label) + if ref is None: + return None + distances.append(ref) + pick = max(range(len(options)), key=lambda i: float(values[i])) + return distances[pick] - min(distances) + + +def score_arm( + model: TreeEditDiffusionModel, + records: list[ExampleRecord], + states: list[dict], +) -> dict: + measurable = [e for e in states if e["ref_distance"] is not None] + preds: list[float] = [] + refs: list[float] = [] + regrets: list[float] = [] + n_regret_excluded = 0 + for entry in measurable: + record = records[entry["record_idx"]] + prompt = model._format_context( + record.prompt, + design_md=record.design_md, + slot_contract=entry["inventory"], + ) + value = _model_value(model, prompt, render_statements(entry["statements"])) + preds.append(value) + refs.append(entry["ref_distance"]) + regret = beam_regret(model, record, entry) + if regret is None: + n_regret_excluded += 1 + else: + regrets.append(regret) + # Higher value must mean closer => correlation against NEGATIVE distance. + rank = spearman(preds, [-r for r in refs]) + conc = concordance(preds, [-r for r in refs]) + targets = [1.0 - min(r, EVAL_MAX_DEPTH) / EVAL_MAX_DEPTH for r in refs] + brier = ( + sum((p - t) ** 2 for p, t in zip(preds, targets)) / len(preds) + if preds + else None + ) + by_origin: dict[str, dict] = {} + for origin in ("near_gold", "seed_trajectory"): + sel = [ + (p, r) + for p, r, e in zip(preds, refs, measurable) + if e["origin"] == origin + ] + if sel: + by_origin[origin] = { + "n": len(sel), + "rank_correlation": spearman( + [p for p, _ in sel], [-r for _, r in sel] + ), + } + return { + "n_measurable": len(measurable), + "rank_correlation": rank, + "concordance": conc, + "brier": brier, + "calibration_bins": calibration_bins(preds, refs), + "beam_regret_mean": (sum(regrets) / len(regrets)) if regrets else None, + "beam_regret_n": len(regrets), + "beam_regret_excluded": n_regret_excluded, + "by_origin": by_origin, + } + + +# --- report --------------------------------------------------------------------- + + +def build_report( + records: list[ExampleRecord], + *, + steps: int, + batch_size: int, + seed: int, +) -> dict: + space = TreeEditSpace() + states = build_eval_states(records, space) + # Oracle labels are arm-independent: label once, share across arms (the + # oracle cache is keyed by grammar/action versions + AST hashes). + oracle = label_eval_states(states, space) + arms: dict[str, dict] = {} + for arm, mode in (("arm_a", "mutation_count"), ("arm_b", "bounded_distance")): + model = train_arm(records, mode, steps=steps, batch_size=batch_size, seed=seed) + metrics = score_arm(model, records, states) + arms[arm] = { + "value_label_mode": mode, + "metrics": metrics, + "training_metrics": model.last_training_metrics, + } + a, b = arms["arm_a"]["metrics"], arms["arm_b"]["metrics"] + + def _delta(key: str, reverse: bool = False) -> float | None: + if a.get(key) is None or b.get(key) is None: + return None + return (a[key] - b[key]) if reverse else (b[key] - a[key]) + + improvements = { + "rank_correlation": _delta("rank_correlation"), + "concordance": _delta("concordance"), + "beam_regret": _delta("beam_regret_mean", reverse=True), + "brier": _delta("brier", reverse=True), + } + rank_ok = ( + improvements["rank_correlation"] is not None + and improvements["rank_correlation"] + >= PREREGISTERED_THRESHOLDS["rank_correlation_improvement_min"] + ) + regret_ok = ( + improvements["beam_regret"] is not None + and improvements["beam_regret"] + >= PREREGISTERED_THRESHOLDS["beam_regret_improvement_min"] + ) + verdict = "adopted" if (rank_ok and regret_ok) else "rejected" + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-308", + "question": ( + "Do bounded-distance oracle value labels beat mutation-count value " + "labels at equal fixture budget (rank correlation + beam regret)?" + ), + "preregistered_thresholds": PREREGISTERED_THRESHOLDS, + "config": { + "steps": steps, + "batch_size": batch_size, + "seed": seed, + "n_records": len(records), + "eval_max_depth": EVAL_MAX_DEPTH, + "eval_node_budget": EVAL_NODE_BUDGET, + }, + "oracle_cost": oracle, + "arms": arms, + "improvements": improvements, + "threshold_checks": { + "rank_correlation_ok": rank_ok, + "beam_regret_ok": regret_ok, + }, + "verdict": verdict, + "honesty": ( + "Fixture-scale matched arms; UNKNOWN/unbounded states are excluded " + "from distance-referenced metrics and counted, never coerced. A " + "fixture verdict is wiring/label evidence, not a production ship " + "claim." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm308_distance_value", + "harness.experiments.slm299_edit_reachability", + ) + return payload + + +def render_markdown(payload: dict) -> str: + a = payload["arms"]["arm_a"]["metrics"] + b = payload["arms"]["arm_b"]["metrics"] + imp = payload["improvements"] + thr = payload["preregistered_thresholds"] + + def fmt(x: object) -> str: + return "n/a" if x is None else (f"{x:.4f}" if isinstance(x, float) else str(x)) + + lines = [ + "# SLM-308 (LAR2-02): bounded-distance value labels vs mutation-count", + "", + f"**Verdict: `{payload['verdict']}`** (fixture-scale matched arms; not a ship claim)", + "", + "## Preregistered thresholds (locked before results)", + "", + f"- beam regret improvement (A−B) >= {thr['beam_regret_improvement_min']}", + f"- rank correlation improvement (B−A) >= {thr['rank_correlation_improvement_min']}", + f"- rule: {thr['verdict_rule']}", + "", + "## Headline", + "", + "| metric | arm A (mutation_count) | arm B (bounded_distance) | improvement |", + "| --- | --- | --- | --- |", + f"| rank correlation | {fmt(a['rank_correlation'])} | {fmt(b['rank_correlation'])} | {fmt(imp['rank_correlation'])} |", + f"| concordance | {fmt(a['concordance'])} | {fmt(b['concordance'])} | {fmt(imp['concordance'])} |", + f"| Brier | {fmt(a['brier'])} | {fmt(b['brier'])} | {fmt(imp['brier'])} |", + f"| beam regret (mean) | {fmt(a['beam_regret_mean'])} | {fmt(b['beam_regret_mean'])} | {fmt(imp['beam_regret'])} |", + "", + "## Coverage + oracle cost", + "", + ] + oracle = payload["oracle_cost"] + lines.append( + f"- UNKNOWN coverage {oracle['unknown_coverage']:.3f} " + f"({oracle['n_unknown']}/{oracle['n_states']}), measurable " + f"{oracle['n_measurable']}, oracle {oracle['ms_per_state']:.1f} ms/state, " + f"{oracle['cache']['map_nodes']} nodes expanded" + ) + lines += [ + "", + "## Calibration (arm B, distance bins d=0..8+)", + "", + "| d | n | mean predicted value | mean target |", + "| --- | --- | --- | --- |", + ] + for row in b["calibration_bins"]: + lines.append( + f"| {row['distance_bin']} | {row['n']} | " + f"{row['mean_predicted_value']:.3f} | {row['mean_target']:.3f} |" + ) + lines += [ + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=8) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + args = parser.parse_args(argv) + + records = build_fixture_records() + payload = build_report( + records, steps=args.steps, batch_size=args.batch_size, seed=args.seed + ) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"verdict={payload['verdict']} " + f"rank_imp={payload['improvements']['rank_correlation']} " + f"regret_imp={payload['improvements']['beam_regret']} " + f"-> {args.json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm308_distance_oracle.py b/src/slm_training/harnesses/experiments/slm308_distance_oracle.py new file mode 100644 index 000000000..dacfe5c7d --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm308_distance_oracle.py @@ -0,0 +1,398 @@ +"""SLM-308 (LAR2-02): bounded distance oracle over canonical AST fingerprints. + +Replaces mutation-count value labels with an exact/bounded/UNKNOWN cost-to-go +signal for the X22 tree-edit model. The oracle runs a **reverse BFS from the +target state** over alpha-invariant canonical fingerprints (``_canonical_key`` +from SLM-299) using the extended 11-action space's transitions. Because every +SLM-305 action is invertible inside the space, the transition graph is +undirected: enumerating forward children of the target is exactly enumerating +its reverse neighborhood, so BFS depth from the target is the true shortest +edit distance for every state it reaches. + +Three labels, and only three: + +- ``EXACT(d)`` — proven shortest distance ``d``: the BFS reached the state and + BFS layering certifies no shorter path exists. +- ``BOUNDED(lo, hi)`` — proof of bounds only, with a machine-readable reason. + ``lo`` comes from fully explored BFS layers (the state was not within the + deepest fully explored depth); ``hi`` comes from a witness path length + (e.g. the mutation chain that produced a training state) when one is + supplied. An invariant proof yields ``BOUNDED(None, None)`` — the target is + unreachable under the exact action set, which is a *proof*, never a budget + artifact. +- ``UNKNOWN`` — node budget exhausted with a live frontier. UNKNOWN is + explicit and is never conflated with far or unreachable; downstream + consumers (value loss) must exclude it, never coerce it. + +Results are cached keyed by ``(action_schema_version, grammar_version, +inventory_hash, target_ast_hash, state_ast_hash, max_depth, witness)``. The +action schema version is the SLM-299 component version (it watches the edit +space); there is no explicit grammar backend version, so the grammar version +is a sha256 fragment of ``dsl/grammars/openui.lark`` — a grammar or action +change is a guaranteed cache miss. + +The oracle is **training-time only**: it reads the gold target and must never +be reachable from any inference/decode path. +""" + +from __future__ import annotations + +import hashlib +import time +from collections import deque +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Sequence + +from slm_training.harness_core.versioning import component_version +from slm_training.harnesses.experiments.slm299_edit_reachability import ( + _canonical_key, + _check_invariants, + _enumerate_children, + _normalize_inventory, +) +from slm_training.models.tree_edit_diffusion import Statement, TreeEditSpace + +__all__ = [ + "DistanceKind", + "DistanceLabel", + "action_schema_version", + "cache_stats", + "clear_caches", + "distance_to_target", + "effective_distance", + "grammar_version", +] + +EXPERIMENT_ID = "slm308-distance-value" + +_GRAMMAR_PATH = ( + Path(__file__).resolve().parents[2] / "dsl" / "grammars" / "openui.lark" +) + +# Reason codes. +REASON_REACHED = "reached" +REASON_DEPTH_BOUND = "depth_bound" # fully explored to max_depth; not found +REASON_FRONTIER_EXHAUSTED = "frontier_exhausted" # frontier died inside bound +REASON_BUDGET = "budget" # node budget exhausted with a live frontier +REASON_INVARIANT_PREFIX = "invariant:" # + slm299 invariant reason code + + +class DistanceKind(str, Enum): + """Proof status of one state→target distance query.""" + + EXACT = "EXACT" + BOUNDED = "BOUNDED" + UNKNOWN = "UNKNOWN" + + +@dataclass +class DistanceLabel: + """Distance proof for one (state, target) pair. + + ``distance`` is set only for EXACT. ``lo``/``hi`` are set for BOUNDED when + the corresponding bound is proven (``hi=None`` means no finite upper bound + was proven — e.g. an invariant-fired unreachable pair). ``nodes_expanded`` + is the oracle's own search cost for the BFS that decided this label. + """ + + kind: DistanceKind + reason: str + distance: int | None = None + lo: int | None = None + hi: int | None = None + nodes_expanded: int = 0 + cache_hit: bool = False + + def value_target(self, max_depth: int) -> float | None: + """Normalized cost-to-go value target in [0, 1] (higher = closer). + + EXACT ``d`` maps to ``1 - d/max_depth``; BOUNDED with both finite + bounds maps to the flagged midpoint; BOUNDED without finite bounds and + UNKNOWN return None — the state is excluded from value loss, never + coerced. + """ + eff = effective_distance(self) + if eff is None: + return None + return max(0.0, 1.0 - min(eff, float(max_depth)) / float(max_depth)) + + def to_dict(self) -> dict[str, object]: + return { + "kind": self.kind.value, + "reason": self.reason, + "distance": self.distance, + "lo": self.lo, + "hi": self.hi, + "nodes_expanded": self.nodes_expanded, + "cache_hit": self.cache_hit, + } + + +def effective_distance(label: DistanceLabel) -> float | None: + """Comparable distance estimate: exact ``d`` or the flagged bound midpoint. + + None when the label carries no usable finite estimate (UNKNOWN, or + invariant-proven unreachable with no witness). Callers must treat None as + "exclude", never as far/unreachable. + """ + if label.kind is DistanceKind.EXACT: + return float(label.distance) + if label.kind is DistanceKind.BOUNDED and label.lo is not None and label.hi is not None: + return (label.lo + label.hi) / 2.0 + return None + + +def action_schema_version() -> str: + """Version of the action space the oracle enumerates (SLM-299 component).""" + return component_version("harness.experiments.slm299_edit_reachability") + + +def grammar_version() -> str: + """Grammar identity: sha256 fragment of the Lark grammar the parser uses.""" + try: + digest = hashlib.sha256(_GRAMMAR_PATH.read_bytes()).hexdigest() + except OSError: + return "unknown" + return f"sha256:{digest[:16]}" + + +def _ast_hash(statements: list[Statement]) -> str: + return hashlib.sha256(_canonical_key(statements).encode("utf-8")).hexdigest()[:16] + + +def _inventory_hash(inventory: Sequence[str]) -> str: + return hashlib.sha256("\n".join(inventory).encode("utf-8")).hexdigest()[:12] + + +# --- caches ----------------------------------------------------------------- + + +@dataclass +class _TargetMap: + """BFS distance map from one target, plus what the search proved.""" + + dist: dict[str, int] = field(default_factory=dict) + deepest_full_depth: int = -1 # every state at this depth was explored + budget_exhausted: bool = False + frontier_died: bool = False # frontier provably emptied before max_depth + nodes_expanded: int = 0 + + +# target map: (action_ver, grammar_ver, inventory_hash, target_ast_hash, max_depth) +_TARGET_MAPS: dict[tuple[str, ...], _TargetMap] = {} +# label cache: target-map key + (state_ast_hash, witness) +_LABELS: dict[tuple, DistanceLabel] = {} +_STATS = {"label_hits": 0, "label_misses": 0, "map_builds": 0} + + +def cache_stats() -> dict[str, int | float]: + return { + **_STATS, + "target_maps": len(_TARGET_MAPS), + "labels": len(_LABELS), + "map_nodes": sum(m.nodes_expanded for m in _TARGET_MAPS.values()), + } + + +def clear_caches() -> None: + _TARGET_MAPS.clear() + _LABELS.clear() + for key in _STATS: + _STATS[key] = 0 + + +def _build_target_map( + target: list[Statement], + inventory: list[str], + space: TreeEditSpace, + *, + max_depth: int, + node_budget: int, +) -> _TargetMap: + """Layered BFS from the target over canonical fingerprints. + + Transitions are the real extended-space children via SLM-299's + ``_enumerate_children`` applied through ``TreeEditSpace.apply`` — the + analyzed space is the deployed space by construction. Because every action + is invertible in-space, BFS depth from the target is the exact shortest + distance for every reached state. + """ + result = _TargetMap() + target_key = _canonical_key(target) + result.dist[target_key] = 0 + frontier: deque[list[Statement]] = deque([target]) + depth = 0 + while frontier and depth < max_depth: + # Expand one full BFS layer; only fully explored layers raise + # ``deepest_full_depth`` (a partial layer proves no lower bound). + layer_size = len(frontier) + layer_complete = True + for _ in range(layer_size): + if result.nodes_expanded >= node_budget: + result.budget_exhausted = True + layer_complete = False + break + statements = frontier.popleft() + result.nodes_expanded += 1 + # Visited pruning is a pure search-efficiency hook: states already + # keyed are rejected pre-validation inside ``apply``; it never + # changes which distinct states are reachable. + visited = set(result.dist) + for child, _action in _enumerate_children( + space, statements, inventory, mode="extended", visited=visited + ): + key = _canonical_key(child) + if key in result.dist: + continue + result.dist[key] = depth + 1 + frontier.append(child) + if not layer_complete: + break + result.deepest_full_depth = depth + depth += 1 + if not frontier and not result.budget_exhausted: + # Frontier provably died: every reachable state within the + # MAX_STMTS-bounded space was enumerated, so unreached states are + # beyond any depth the space connects — still reported via bounds. + result.frontier_died = True + return result + + +def distance_to_target( + statements: list[Statement], + target_statements: list[Statement], + *, + space: TreeEditSpace, + inventory: Sequence[str] = (), + max_depth: int = 8, + node_budget: int = 600, + upper_bound_witness: int | None = None, +) -> DistanceLabel: + """Distance proof from ``statements`` to ``target_statements``. + + ``upper_bound_witness`` is a proven path length (e.g. the mutation chain + that produced a training state from the target); it only ever tightens + ``hi``, never ``lo``. UNKNOWN is returned on budget exhaustion and is + never conflated with far or unreachable. + """ + versions = (action_schema_version(), grammar_version()) + inv = _normalize_inventory(inventory) + inv_hash = _inventory_hash(inv) + state_hash = _ast_hash(statements) + target_hash = _ast_hash(target_statements) + map_key = (*versions, inv_hash, target_hash, str(max_depth)) + label_key = (map_key, state_hash, upper_bound_witness, node_budget) + + cached = _LABELS.get(label_key) + if cached is not None: + _STATS["label_hits"] += 1 + return DistanceLabel( + kind=cached.kind, + reason=cached.reason, + distance=cached.distance, + lo=cached.lo, + hi=cached.hi, + nodes_expanded=cached.nodes_expanded, + cache_hit=True, + ) + _STATS["label_misses"] += 1 + + if state_hash == target_hash: + label = DistanceLabel( + kind=DistanceKind.EXACT, reason=REASON_REACHED, distance=0 + ) + _LABELS[label_key] = label + return label + + # Structural impossibility proofs over the EXACT extended action set + # (container_add is a real capability of the extended space). + fired = _check_invariants( + target_statements, + statements, + inv, + space, + frozenset({"container_add"}), + extended=True, + ) + if fired is not None: + label = DistanceLabel( + kind=DistanceKind.BOUNDED, + reason=f"{REASON_INVARIANT_PREFIX}{fired}", + lo=None, + hi=None, + ) + _LABELS[label_key] = label + return label + + target_map = _TARGET_MAPS.get(map_key) + if target_map is None: + _STATS["map_builds"] += 1 + target_map = _build_target_map( + target_statements, + inv, + space, + max_depth=max_depth, + node_budget=node_budget, + ) + _TARGET_MAPS[map_key] = target_map + + state_key = _canonical_key(statements) + exact = target_map.dist.get(state_key) + if exact is not None: + label = DistanceLabel( + kind=DistanceKind.EXACT, + reason=REASON_REACHED, + distance=exact, + nodes_expanded=target_map.nodes_expanded, + ) + elif target_map.budget_exhausted: + # Live frontier when the budget cut the search: nothing is proven + # about this state beyond the fully explored layers, and a witness + # alone never upgrades an UNKNOWN to bounded-exact. + if upper_bound_witness is not None and target_map.deepest_full_depth >= 0: + label = DistanceLabel( + kind=DistanceKind.BOUNDED, + reason=REASON_BUDGET, + lo=target_map.deepest_full_depth + 2, + hi=upper_bound_witness, + nodes_expanded=target_map.nodes_expanded, + ) + else: + label = DistanceLabel( + kind=DistanceKind.UNKNOWN, + reason=REASON_BUDGET, + nodes_expanded=target_map.nodes_expanded, + ) + else: + if target_map.frontier_died: + # Complete proof: the target's connected component was fully + # enumerated and the state is not in it — unreachable, with the + # same status as an invariant proof (no finite bounds). + label = DistanceLabel( + kind=DistanceKind.BOUNDED, + reason=REASON_FRONTIER_EXHAUSTED, + lo=None, + hi=None, + nodes_expanded=target_map.nodes_expanded, + ) + else: + # Depth bound cut a live frontier: distances up to max_depth were + # enumerated, so this state's distance is provably > max_depth. + label = DistanceLabel( + kind=DistanceKind.BOUNDED, + reason=REASON_DEPTH_BOUND, + lo=max_depth + 1, + hi=upper_bound_witness, + nodes_expanded=target_map.nodes_expanded, + ) + _LABELS[label_key] = label + return label + + +def timed_distance_to_target(**kwargs) -> tuple[DistanceLabel, float]: + """``distance_to_target`` plus wall-clock milliseconds (oracle cost evidence).""" + start = time.perf_counter() + label = distance_to_target(**kwargs) + return label, (time.perf_counter() - start) * 1000.0 diff --git a/src/slm_training/models/tree_edit_diffusion.py b/src/slm_training/models/tree_edit_diffusion.py index 13ffc430d..9d973f5b1 100644 --- a/src/slm_training/models/tree_edit_diffusion.py +++ b/src/slm_training/models/tree_edit_diffusion.py @@ -876,6 +876,41 @@ class TreeEditDiffusionConfig: schema_in_context: bool = False slot_contract_in_context: bool = True seed: int = 0 + # SLM-308 (LAR2-02): value supervision mode. "bounded_distance" (default + # for NEW trainings) labels values with the SLM-308 distance oracle's + # normalized cost-to-go plus pairwise parent/improving-child ranking; + # "mutation_count" keeps the historical 1 - applied/(max_chain+1) labels. + # Checkpoints written before this field existed load as "mutation_count" + # (see ``from_checkpoint``) for behavior parity. + value_label_mode: str = "bounded_distance" + pairwise_progress_margin: float = 0.1 + + +# SLM-308: oracle depth/budget used for training-time value labels. The +# budget is shallow by design (extended-space enumeration is parser-backed +# and expensive); states beyond the explored layers get witness-bounded +# BOUNDED labels or are excluded (UNKNOWN), never coerced. +VALUE_ORACLE_MAX_DEPTH = 8 +VALUE_ORACLE_NODE_BUDGET = 8 + + +def pairwise_progress_loss( + parent_values: torch.Tensor, + child_values: torch.Tensor, + *, + margin: float = 0.1, +) -> torch.Tensor: + """Margin ranking: an improving child (oracle-proven closer to the target) + must score at least ``margin`` higher on value than its parent. + + Higher value = closer to target. Pairs are pre-filtered by the caller to + strictly-improving, oracle-comparable pairs only — ties and unmeasurable + (UNKNOWN / unbounded) pairs never enter here, so several comparably-close + states are never forced into a strict order. + """ + if parent_values.numel() == 0: + return parent_values.new_zeros(()) + return F.relu(parent_values - child_values + margin).mean() class TreeEditPolicy(nn.Module): @@ -1029,15 +1064,45 @@ def _state_batch(self, sources: list[str]) -> torch.Tensor: # --- training --------------------------------------------------------- + def _distance_label(self, statements, target, inventory, witness: int): + """Training-time ONLY gold-distance label (SLM-308 oracle). + + Imported lazily: the oracle reads the gold target and must never be + reachable from any decode path (see the SLM-308 no-gold-at-inference + audit test). + """ + from slm_training.harnesses.experiments.slm308_distance_oracle import ( + distance_to_target, + ) + + return distance_to_target( + statements, + target, + space=self.space, + inventory=inventory, + max_depth=VALUE_ORACLE_MAX_DEPTH, + node_budget=VALUE_ORACLE_NODE_BUDGET, + upper_bound_witness=witness, + ) + def forward(self, batch: list[ExampleRecord]) -> float: return float(self.training_loss(batch).detach().cpu()) def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: + if self.config.value_label_mode not in {"mutation_count", "bounded_distance"}: + raise ValueError( + f"unknown value_label_mode {self.config.value_label_mode!r}" + ) + bounded_mode = self.config.value_label_mode == "bounded_distance" prompts: list[str] = [] states: list[str] = [] targets: list[Edit] = [] values: list[float] = [] + value_mask: list[bool] = [] + pair_rows: list[tuple[int, str]] = [] # (parent row, child source) skipped = 0 + n_bounded = 0 + n_unknown_excluded = 0 for record in batch: source = (record.openui or "").strip() statements = parse_statements(source) if source else None @@ -1054,20 +1119,24 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: slot_contract=inventory, ) if self._rng.random() < 0.2: - # Clean state: the correct move is STOP with full value. + # Clean state: the correct move is STOP with full value + # (oracle distance 0 in bounded mode, so the modes agree). prompts.append(prompt) states.append(source) targets.append(Edit(ACTION_STOP)) values.append(1.0) + value_mask.append(True) continue k = self._rng.randint(1, self.config.max_chain) current = statements + prev = None inverse: Edit | None = None applied = 0 for _ in range(k): step = self.space.sample_mutation(current, inventory, self._rng) if step is None: break + prev = current current, inverse = step applied += 1 if inverse is None: @@ -1076,16 +1145,61 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: prompts.append(prompt) states.append(render_statements(current)) targets.append(inverse) - values.append(1.0 - applied / float(self.config.max_chain + 1)) + if not bounded_mode: + values.append(1.0 - applied / float(self.config.max_chain + 1)) + value_mask.append(True) + continue + # SLM-308: normalized cost-to-go from the bounded distance oracle. + # UNKNOWN / unbounded states are excluded from value loss, never + # coerced; the mutation chain length is a proven witness upper + # bound (the inverse edits walk back to gold). + label = self._distance_label(current, statements, inventory, applied) + target_value = label.value_target(VALUE_ORACLE_MAX_DEPTH) + if target_value is None: + values.append(0.0) + value_mask.append(False) + n_unknown_excluded += 1 + else: + values.append(target_value) + value_mask.append(True) + if label.kind.value == "BOUNDED": + n_bounded += 1 + if prev is not None: + # Pairwise progress: prev is one inverse-edit closer to gold. + # The pair enters the ranking loss only when the oracle proves + # a strict improvement on comparable (finite) estimates. + from slm_training.harnesses.experiments.slm308_distance_oracle import ( # noqa: E501 + effective_distance, + ) + + child_label = self._distance_label( + prev, statements, inventory, applied - 1 + ) + d_parent = effective_distance(label) + d_child = effective_distance(child_label) + if ( + d_parent is not None + and d_child is not None + and d_child < d_parent + ): + pair_rows.append((len(states) - 1, render_statements(prev))) if not states: return torch.zeros((), device=self.device_name, requires_grad=True) + n_main = len(states) + pair_child_rows = list(range(n_main, n_main + len(pair_rows))) + states.extend(child_source for _, child_source in pair_rows) ctx, ctx_pad = self._encode_context(prompts) + if pair_rows: + # Pairwise-ranking child rows reuse their parent's prompt context. + ctx_rows = [row for row, _ in pair_rows] + ctx = torch.cat([ctx, ctx[ctx_rows]], dim=0) + ctx_pad = torch.cat([ctx_pad, ctx_pad[ctx_rows]], dim=0) out = self.policy( self._state_batch(states), self.tokenizer.pad_id, ctx, ctx_pad ) device = self.device_name action_t = torch.tensor([e.action for e in targets], device=device) - loss = F.cross_entropy(out["action"], action_t) + loss = F.cross_entropy(out["action"][:n_main], action_t) losses = {"action": float(loss.detach().cpu())} stmt_rows = [i for i, e in enumerate(targets) if e.action != ACTION_STOP] if stmt_rows: @@ -1130,9 +1244,21 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: loss = loss + slot_loss losses["slot"] = float(slot_loss.detach().cpu()) value_t = torch.tensor(values, device=device, dtype=out["value"].dtype) - value_loss = F.mse_loss(out["value"], value_t) - loss = loss + value_loss - losses["value"] = float(value_loss.detach().cpu()) + mask_t = torch.tensor(value_mask, device=device, dtype=torch.bool) + if bool(mask_t.any()): + value_loss = F.mse_loss(out["value"][:n_main][mask_t], value_t[mask_t]) + loss = loss + value_loss + losses["value"] = float(value_loss.detach().cpu()) + if pair_rows: + pair_loss = pairwise_progress_loss( + out["value"][torch.tensor([row for row, _ in pair_rows], device=device)], + out["value"][torch.tensor(pair_child_rows, device=device)], + margin=self.config.pairwise_progress_margin, + ) + loss = loss + pair_loss + losses["pairwise_progress"] = float(pair_loss.detach().cpu()) + losses["value_bounded"] = float(n_bounded) + losses["value_unknown_excluded"] = float(n_unknown_excluded) losses["skipped"] = float(skipped) self.last_training_metrics = losses return loss @@ -1444,7 +1570,11 @@ def from_checkpoint( "slm_training.models.checkpoint_migrate.migrate_tree_edit_checkpoint" ) tokenizer = OpenUITokenizer.load(path.with_suffix(".tokenizer.json")) - config = TreeEditDiffusionConfig(**payload["config"]) + config_payload = dict(payload["config"]) + # SLM-308: checkpoints written before value_label_mode existed were + # trained with mutation-count labels — preserve that behavior exactly. + config_payload.setdefault("value_label_mode", "mutation_count") + config = TreeEditDiffusionConfig(**config_payload) model = cls(tokenizer, config=config, device=device) model.load_state_dict(payload["state_dict"], strict=True) return model diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 0397d95b9..7a3a7c3bd 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -665,7 +665,7 @@ ] }, "evals.agentv": { - "version": "v3", + "version": "v4", "kind": "harness", "paths": [ "src/slm_training/evals/agentv.py", @@ -673,6 +673,11 @@ "tests/test_evals/test_agentv.py" ], "history": [ + { + "version": "v4", + "date": "2026-07-24", + "note": "Replace the optional Node LangSmith SDK with bounded native fetch so Vercel does not bundle CLI-only dependencies" + }, { "version": "v3", "date": "2026-07-24", @@ -4475,7 +4480,7 @@ ] }, "harness.experiments.slm299_edit_reachability": { - "version": "v2", + "version": "v3", "kind": "harness", "paths": [ "src/slm_training/harnesses/experiments/slm299_edit_reachability.py", @@ -4489,6 +4494,11 @@ "tests/test_models/test_tree_edit_diffusion.py" ], "history": [ + { + "version": "v3", + "date": "2026-07-24", + "note": "SLM-308: tree_edit_diffusion gains value_label_mode (bounded_distance default for new trainings, mutation_count parity for pre-field checkpoints), oracle-backed normalized cost-to-go value targets with UNKNOWN exclusion, and pairwise parent/improving-child progress loss; edit space and reachability machinery unchanged" + }, { "version": "v2", "date": "2026-07-24", @@ -4557,6 +4567,24 @@ } ] }, + "harness.experiments.slm308_distance_value": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm308_distance_oracle.py", + "scripts/run_slm308_distance_value.py", + "tests/test_harnesses/experiments/test_slm308_distance_value.py", + "docs/design/iter-slm308-distance-value-20260724.json", + "docs/design/iter-slm308-distance-value-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-308 LAR2-02 bounded reverse-BFS distance oracle (EXACT/BOUNDED/UNKNOWN) over canonical AST fingerprints, distance value supervision + pairwise progress ranking in the X22 tree-edit model, matched fixture experiment with preregistered thresholds" + } + ] + }, "harness.model_build.eval": { "version": "v54", "kind": "harness", diff --git a/tests/test_harnesses/experiments/test_slm308_distance_value.py b/tests/test_harnesses/experiments/test_slm308_distance_value.py new file mode 100644 index 000000000..472a39f7c --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm308_distance_value.py @@ -0,0 +1,357 @@ +"""SLM-308 (LAR2-02): distance oracle, value supervision, and matched-experiment tests.""" + +from __future__ import annotations + +import inspect + +import pytest +import torch + +import slm_training.harnesses.experiments.slm308_distance_oracle as oracle +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm308_distance_oracle import ( + DistanceKind, + clear_caches, + distance_to_target, + effective_distance, +) +from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + pairwise_progress_loss, + parse_statements, +) +from scripts.run_slm308_distance_value import ( + build_eval_states, + build_fixture_records, + build_report, + concordance, + render_markdown, + spearman, +) + +SEED = 'root = Stack([], "column")' +GOLD = 'root = Stack([cta], "column")\ncta = Button(":cta.label")' + + +@pytest.fixture() +def space(): + from slm_training.models.tree_edit_diffusion import TreeEditSpace + + return TreeEditSpace() + + +@pytest.fixture(autouse=True) +def _clean_oracle_caches(): + clear_caches() + yield + clear_caches() + + +# (a) EXACT / BOUNDED / UNKNOWN distinction --------------------------------- + + +def test_exact_distance_small_cases(space) -> None: + seed = parse_statements(SEED) + gold = parse_statements(GOLD) + label = distance_to_target( + seed, gold, space=space, inventory=[":cta.label"], node_budget=4 + ) + assert label.kind is DistanceKind.EXACT + assert label.distance == 1 + assert label.value_target(8) == pytest.approx(1.0 - 1 / 8) + # Identity is distance 0; the graph is undirected so reverse is exact too. + ident = distance_to_target(gold, gold, space=space, inventory=[":x"]) + assert ident.kind is DistanceKind.EXACT and ident.distance == 0 + rev = distance_to_target( + gold, seed, space=space, inventory=[":cta.label"], node_budget=4 + ) + assert rev.kind is DistanceKind.EXACT and rev.distance == 1 + + +def test_invariant_proof_is_bounded_without_finite_bounds(space) -> None: + seed = parse_statements(SEED) + bad = parse_statements('root = Stack([cta], "column")\ncta = Button(":zzz")') + label = distance_to_target(seed, bad, space=space, inventory=[":cta.label"]) + assert label.kind is DistanceKind.BOUNDED + assert label.reason == "invariant:needs_slot_rebind" + assert label.lo is None and label.hi is None + # Unbounded proofs are excluded from value supervision, never coerced. + assert label.value_target(8) is None + assert effective_distance(label) is None + + +def test_budget_exhaustion_is_explicit_unknown(space) -> None: + seed = parse_statements(SEED) + gold = parse_statements(GOLD) + # node_budget=1: the target layer expands, then the budget cuts layer 1 + # with a live frontier. UNKNOWN, never conflated with far/unreachable. + label = distance_to_target( + seed, gold, space=space, inventory=[":cta.label"], + max_depth=8, node_budget=1, + ) + # The seed itself is a depth-1 child of the target, so it IS reached; + # use a state that is not adjacent for the UNKNOWN check. + assert label.kind is DistanceKind.EXACT + far = parse_statements( + 'root = Stack([a, b], "column")\na = TextContent(":x")\nb = Button(":y")' + ) + # Both states' leaf slots must be in inventory, or the invariant proof + # (not the budget path) decides the label. + inv = [":x", ":y", ":cta.label"] + label = distance_to_target( + far, gold, space=space, inventory=inv, + max_depth=8, node_budget=1, + ) + assert label.kind is DistanceKind.UNKNOWN + assert label.reason == "budget" + assert label.value_target(8) is None + # A proven witness path upgrades UNKNOWN to BOUNDED(lo, hi) — the bounds + # are proofs (fully explored layers + witness), still never coerced exact. + bounded = distance_to_target( + far, gold, space=space, inventory=inv, + max_depth=8, node_budget=1, upper_bound_witness=3, + ) + assert bounded.kind is DistanceKind.BOUNDED + assert bounded.reason == "budget" + assert (bounded.lo, bounded.hi) == (2, 3) + assert bounded.value_target(8) == pytest.approx(1.0 - 2.5 / 8) + + +def test_depth_bound_is_bounded_not_unknown(space) -> None: + gold = parse_statements(GOLD) + far = parse_statements( + 'root = Stack([a, b], "column")\na = TextContent(":x")\nb = Button(":y")' + ) + # max_depth=0: nothing explored, budget intact, frontier still live. + label = distance_to_target( + far, gold, space=space, inventory=[":x", ":y", ":cta.label"], + max_depth=0, node_budget=1, upper_bound_witness=4, + ) + assert label.kind is DistanceKind.BOUNDED + assert label.reason == "depth_bound" + assert (label.lo, label.hi) == (1, 4) + + +# (b) cache identity ---------------------------------------------------------- + + +def test_cache_hit_and_version_change_misses(space, monkeypatch) -> None: + seed = parse_statements(SEED) + gold = parse_statements(GOLD) + kwargs = dict(space=space, inventory=[":cta.label"], node_budget=4) + first = distance_to_target(seed, gold, **kwargs) + assert not first.cache_hit + second = distance_to_target(seed, gold, **kwargs) + assert second.cache_hit and second.kind is first.kind + # Action-schema version change => guaranteed miss. + monkeypatch.setattr(oracle, "action_schema_version", lambda: "v999") + third = distance_to_target(seed, gold, **kwargs) + assert not third.cache_hit + # Grammar version change => guaranteed miss. + monkeypatch.setattr(oracle, "grammar_version", lambda: "sha256:deadbeef") + fourth = distance_to_target(seed, gold, **kwargs) + assert not fourth.cache_hit + + +def test_grammar_version_is_grammar_file_hash() -> None: + assert oracle.grammar_version().startswith("sha256:") + assert oracle.action_schema_version().startswith("v") + + +# (c) pairwise progress loss --------------------------------------------------- + + +def test_pairwise_progress_loss_improving_worsening_tie() -> None: + # Improving pair with margin satisfied: child scores >= parent + margin. + zero = pairwise_progress_loss( + torch.tensor([0.5]), torch.tensor([0.7]), margin=0.1 + ) + assert float(zero) == 0.0 + # Worsening pair (child below parent): full violation + margin. + loss = pairwise_progress_loss( + torch.tensor([0.6]), torch.tensor([0.4]), margin=0.1 + ) + assert float(loss) == pytest.approx(0.3) + # Tie inside the margin: only the margin remains. + tie = pairwise_progress_loss( + torch.tensor([0.5]), torch.tensor([0.5]), margin=0.1 + ) + assert float(tie) == pytest.approx(0.1) + # Empty pair set is a no-op zero, never an error. + empty = pairwise_progress_loss(torch.zeros(0), torch.zeros(0)) + assert float(empty) == 0.0 + # Gradient flows to both sides. + parent = torch.tensor([0.5], requires_grad=True) + child = torch.tensor([0.4], requires_grad=True) + pairwise_progress_loss(parent, child, margin=0.1).backward() + assert parent.grad is not None and child.grad is not None + + +# (d) no gold distance at inference -------------------------------------------- + +_RECORDS = [ + ExampleRecord( + id="r0", + prompt="hero with cta", + openui=GOLD, + placeholders=[":cta.label"], + split="train", + ), + ExampleRecord( + id="r1", + prompt="simple text", + openui='root = Stack([t], "column")\nt = TextContent(":body")', + placeholders=[":body"], + split="train", + ), +] + + +def _tiny_model(mode: str = "bounded_distance") -> TreeEditDiffusionModel: + config = TreeEditDiffusionConfig( + value_label_mode=mode, + d_model=32, + n_heads=2, + denoiser_layers=1, + context_layers=1, + beam_width=2, + expand_per_state=2, + max_search_steps=2, + max_chain=2, + ) + return TreeEditDiffusionModel.from_records(_RECORDS, config=config, device="cpu") + + +def test_decode_never_calls_the_oracle(monkeypatch) -> None: + # Runtime audit: any oracle call from the decode path fails loudly. + def _boom(*args, **kwargs): + raise AssertionError("oracle reached from inference path") + + monkeypatch.setattr(oracle, "distance_to_target", _boom) + model = _tiny_model() + text = model.generate("hero with cta") + assert isinstance(text, str) + # Static audit: the decode functions carry no oracle reference. + for fn_name in ("_decode_one", "_enumerate_edits", "_seed_state", "generate_batch_requests", "generate"): + source = inspect.getsource(getattr(TreeEditDiffusionModel, fn_name)) + assert "distance_to_target" not in source + assert "slm308" not in source + + +# (e) value supervision modes ---------------------------------------------------- + + +def test_training_loss_bounded_mode_uses_oracle_and_pairs() -> None: + model = _tiny_model("bounded_distance") + loss = model.training_loss(_RECORDS) + assert torch.isfinite(loss.detach()) + metrics = model.last_training_metrics + assert "value" in metrics + assert "value_bounded" in metrics + assert "value_unknown_excluded" in metrics + # Fixture chains are 1-2 edits from gold: at least one oracle-labeled + # value and one strictly-improving pair must exist. + assert metrics["value_bounded"] + 1 >= 1 + assert "pairwise_progress" in metrics + + +def test_training_loss_mutation_count_mode_matches_legacy() -> None: + model = _tiny_model("mutation_count") + loss = model.training_loss(_RECORDS) + assert torch.isfinite(loss.detach()) + metrics = model.last_training_metrics + # Legacy mode: every row carries a value, no oracle involvement, no pairs. + assert metrics["value_unknown_excluded"] == 0.0 + assert metrics["value_bounded"] == 0.0 + assert "pairwise_progress" not in metrics + + +def test_unknown_states_are_excluded_from_value_loss(monkeypatch) -> None: + model = _tiny_model("bounded_distance") + + class _Unknown: + kind = DistanceKind.UNKNOWN + + def value_target(self, max_depth: int): + return None + + monkeypatch.setattr( + TreeEditDiffusionModel, "_distance_label", lambda *a, **k: _Unknown() + ) + loss = model.training_loss(_RECORDS) + assert torch.isfinite(loss.detach()) + metrics = model.last_training_metrics + assert metrics["value_unknown_excluded"] >= 1.0 + + +# (f) checkpoint parity ----------------------------------------------------------- + + +def test_old_checkpoints_default_to_mutation_count(tmp_path) -> None: + model = _tiny_model("bounded_distance") + path = tmp_path / "ckpt.pt" + model.save(path) + payload = torch.load(path, map_location="cpu", weights_only=False) + assert payload["config"]["value_label_mode"] == "bounded_distance" + # Round-trip preserves the new mode. + loaded = TreeEditDiffusionModel.from_checkpoint(path) + assert loaded.config.value_label_mode == "bounded_distance" + # A format-2 checkpoint written before the field existed loads as + # mutation_count (behavior parity). + del payload["config"]["value_label_mode"] + torch.save(payload, path) + legacy = TreeEditDiffusionModel.from_checkpoint(path) + assert legacy.config.value_label_mode == "mutation_count" + + +def test_new_config_defaults_to_bounded_distance() -> None: + assert TreeEditDiffusionConfig().value_label_mode == "bounded_distance" + + +# (g) metric helpers + matched-experiment determinism ------------------------------ + + +def test_spearman_and_concordance() -> None: + assert spearman([1, 2, 3, 4], [1, 2, 3, 4]) == pytest.approx(1.0) + assert spearman([1, 2, 3, 4], [4, 3, 2, 1]) == pytest.approx(-1.0) + assert spearman([1, 1, 1], [1, 2, 3]) is None + assert concordance([0.1, 0.5, 0.9], [0.1, 0.5, 0.9]) == pytest.approx(1.0) + assert concordance([0.9, 0.5, 0.1], [0.1, 0.5, 0.9]) == pytest.approx(0.0) + + +def test_eval_states_are_deterministic(space) -> None: + records = build_fixture_records()[:2] + first = build_eval_states(records, space) + second = build_eval_states(records, space) + assert len(first) == len(second) > 0 + for a, b in zip(first, second): + assert a["origin"] == b["origin"] + assert a["witness"] == b["witness"] + from slm_training.models.tree_edit_diffusion import render_statements + + assert render_statements(a["statements"]) == render_statements(b["statements"]) + assert {e["origin"] for e in first} == {"near_gold", "seed_trajectory"} + + +@pytest.mark.training +def test_matched_experiment_is_deterministic() -> None: + import copy + + records = build_fixture_records()[:2] + first = build_report(records, steps=2, batch_size=2, seed=0) + second = build_report(records, steps=2, batch_size=2, seed=0) + md = render_markdown(first) + stripped = [] + for payload in (first, second): + view = copy.deepcopy(payload) + view.pop("version_stamp") + # Wall-clock oracle cost fields are evidence, not identity. + view["oracle_cost"].pop("wall_ms_total") + view["oracle_cost"].pop("ms_per_state") + stripped.append(view) + assert stripped[0] == stripped[1] + assert first["verdict"] in {"adopted", "rejected"} + assert first["preregistered_thresholds"]["beam_regret_improvement_min"] == 0.05 + md = render_markdown(first) + assert "Preregistered thresholds" in md From d8f35563d231e3b404b0fb0e213d68aee1b00a9d Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:11:37 -0500 Subject: [PATCH 06/10] SLM-310: align inverse-edit supervision with seed-originated demand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Proposal instrumentation: per enumerated candidate — action, factor score, applicability, rejection reason (23 machine-readable codes), budget consumption; deterministic order; additive evidence keys. - Distribution audit: training-target vs decode-demand action distributions by source/suite; dead-candidate rate ~0.98, applicable-ADD recall 0.10 baseline; preregistered reweighting rule. - corruption_action_distribution knob (gold corpus untouched; default off = historical uniform; parity tested) and stop_slot_accounting legacy|corrected arm (STOP consumes an expansion slot only when its frozen candidate is retained; deterministic, regression-tested). - Matched 2x2 (ADD-balanced x STOP arms, isolated levers): ADD target share 0.177 -> 0.328 (T1 +0.150 >= 0.10 both STOP arms), recall 0.10 -> 0.16, corrected STOP budget <= legacy; verdicts adopted per preregistered rules; loss reweighting deferred to preserve isolation. - 10 new tests; slm299 component v4; slm310 component v1. --- ...iter-slm310-action-alignment-20260724.json | 3792 +++++++++++++++++ .../iter-slm310-action-alignment-20260724.md | 38 + scripts/run_slm310_action_alignment.py | 481 +++ .../experiments/slm310_action_alignment.py | 330 ++ .../models/tree_edit_diffusion.py | 337 +- src/slm_training/resources/versions.json | 46 +- .../test_slm310_action_alignment.py | 405 ++ 7 files changed, 5343 insertions(+), 86 deletions(-) create mode 100644 docs/design/iter-slm310-action-alignment-20260724.json create mode 100644 docs/design/iter-slm310-action-alignment-20260724.md create mode 100644 scripts/run_slm310_action_alignment.py create mode 100644 src/slm_training/harnesses/experiments/slm310_action_alignment.py create mode 100644 tests/test_harnesses/experiments/test_slm310_action_alignment.py diff --git a/docs/design/iter-slm310-action-alignment-20260724.json b/docs/design/iter-slm310-action-alignment-20260724.json new file mode 100644 index 000000000..0dd364d00 --- /dev/null +++ b/docs/design/iter-slm310-action-alignment-20260724.json @@ -0,0 +1,3792 @@ +{ + "experiment": "slm310-action-alignment", + "issue": "SLM-310", + "question": "Does ADD-balanced corruption sampling close the train/demand action gap, and does corrected STOP-slot accounting improve search budget use \u2014 each lever isolated at fixture scale?", + "preregistered_thresholds": { + "t1_add_target_share_gain_min": 0.1, + "t2_valid_final_no_regression": true, + "t3_stop_budget_not_worse_and_valid_final_no_regression": true, + "t4_lever_isolation_structural": true, + "verdict_rule": "per lever: adopted iff its thresholds hold in both cells of the matched pair, else rejected; levers are never combined" + }, + "config": { + "steps": 8, + "batch_size": 4, + "seed": 0, + "audit_samples": 400, + "n_records": 8, + "value_label_mode": "mutation_count", + "add_balanced_distribution": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + }, + "cells": { + "cell_a": { + "corruption": null, + "stop": "legacy" + }, + "cell_b": { + "corruption": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + }, + "stop": "legacy" + }, + "cell_c": { + "corruption": null, + "stop": "corrected" + }, + "cell_d": { + "corruption": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + }, + "stop": "corrected" + } + } + }, + "cells": { + "cell_a": { + "levers": { + "corruption": null, + "stop": "legacy" + }, + "metrics": { + "training_targets": { + "samples": 400, + "n_targets": 378, + "counts": { + "STOP": 71, + "REPLACE": 42, + "ADD": 67, + "REMOVE": 69, + "ADD_CONTAINER": 5, + "REMOVE_CONTAINER": 69, + "INSERT_SUBTREE": 7, + "REPLACE_SUBTREE": 19, + "INSERT_STATEMENT": 0, + "REPLACE_STATEMENT": 7, + "BIND_PLACEHOLDER": 22 + }, + "shares": { + "STOP": 0.18783068783068782, + "REPLACE": 0.1111111111111111, + "ADD": 0.17724867724867724, + "REMOVE": 0.18253968253968253, + "ADD_CONTAINER": 0.013227513227513227, + "REMOVE_CONTAINER": 0.18253968253968253, + "INSERT_SUBTREE": 0.018518518518518517, + "REPLACE_SUBTREE": 0.05026455026455026, + "INSERT_STATEMENT": 0.0, + "REPLACE_STATEMENT": 0.018518518518518517, + "BIND_PLACEHOLDER": 0.0582010582010582 + }, + "inverse_action_weights": null + }, + "decode_demand": { + "overall": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 11092, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + }, + "by_suite": { + "train": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 0, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + } + }, + "by_source": { + "fixture_00": { + "visited": { + "BIND_PLACEHOLDER": 576, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 15, + "REPLACE": 3, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 297, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 9, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 185, + "invalid_result": 1, + "leaf_container_mismatch": 1, + "not_bindable": 252, + "not_canonical_subtree": 288, + "not_container": 15, + "not_leaf_component": 144, + "not_removable": 6, + "target_not_canonical": 2 + }, + "visited_total": 918, + "dead_total": 894, + "dead_candidate_rate": 0.9738562091503268, + "visited_shares": { + "BIND_PLACEHOLDER": 0.6274509803921569, + "INSERT_STATEMENT": 0.013071895424836602, + "REMOVE": 0.007625272331154684, + "REMOVE_CONTAINER": 0.016339869281045753, + "REPLACE": 0.0032679738562091504, + "REPLACE_STATEMENT": 0.002178649237472767, + "REPLACE_SUBTREE": 0.3235294117647059, + "STOP": 0.006535947712418301 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_01": { + "visited": { + "ADD": 2, + "BIND_PLACEHOLDER": 648, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 17, + "REMOVE": 3, + "REMOVE_CONTAINER": 14, + "REPLACE": 2, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 1022, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 3, + "REPLACE_SUBTREE": 230, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 17, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 7, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 407, + "invalid_result": 73, + "leaf_container_mismatch": 1, + "not_bindable": 432, + "not_canonical_subtree": 720, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 2, + "parent_or_comp_precondition": 17, + "target_not_canonical": 2 + }, + "visited_total": 1737, + "dead_total": 1701, + "dead_candidate_rate": 0.9792746113989638, + "visited_shares": { + "ADD": 0.0011514104778353484, + "BIND_PLACEHOLDER": 0.37305699481865284, + "INSERT_STATEMENT": 0.010362694300518135, + "INSERT_SUBTREE": 0.00978698906160046, + "REMOVE": 0.0017271157167530224, + "REMOVE_CONTAINER": 0.008059873344847437, + "REPLACE": 0.0011514104778353484, + "REPLACE_STATEMENT": 0.0011514104778353484, + "REPLACE_SUBTREE": 0.588370754173863, + "STOP": 0.0051813471502590676 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_02": { + "visited": { + "BIND_PLACEHOLDER": 504, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1431, + "dead_total": 1407, + "dead_candidate_rate": 0.9832285115303984, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3522012578616352, + "INSERT_STATEMENT": 0.008385744234800839, + "REMOVE": 0.004891684136967156, + "REMOVE_CONTAINER": 0.007686932215234102, + "REPLACE": 0.011879804332634521, + "REPLACE_STATEMENT": 0.0041928721174004195, + "REPLACE_SUBTREE": 0.6065688329839273, + "STOP": 0.0041928721174004195 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_03": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 8, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 72, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 75, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 8 + }, + "visited_total": 1397, + "dead_total": 1373, + "dead_candidate_rate": 0.9828203292770222, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3350035790980673, + "INSERT_STATEMENT": 0.008589835361488905, + "REMOVE": 0.005010737294201861, + "REMOVE_CONTAINER": 0.007874015748031496, + "REPLACE": 0.012168933428775949, + "REPLACE_STATEMENT": 0.00572655690765927, + "REPLACE_SUBTREE": 0.6213314244810307, + "STOP": 0.004294917680744453 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_04": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1395, + "dead_total": 1371, + "dead_candidate_rate": 0.9827956989247312, + "visited_shares": { + "BIND_PLACEHOLDER": 0.33548387096774196, + "INSERT_STATEMENT": 0.008602150537634409, + "REMOVE": 0.005017921146953405, + "REMOVE_CONTAINER": 0.007885304659498209, + "REPLACE": 0.012186379928315413, + "REPLACE_STATEMENT": 0.004301075268817204, + "REPLACE_SUBTREE": 0.6222222222222222, + "STOP": 0.004301075268817204 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_05": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 48, + "REMOVE": 4, + "REMOVE_CONTAINER": 9, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 9, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 50, + "target_not_canonical": 4 + }, + "visited_total": 1226, + "dead_total": 1194, + "dead_candidate_rate": 0.9738988580750407, + "visited_shares": { + "ADD": 0.0032626427406199023, + "BIND_PLACEHOLDER": 0.44045676998368677, + "INSERT_STATEMENT": 0.013050570962479609, + "INSERT_SUBTREE": 0.03915171288743882, + "REMOVE": 0.0032626427406199023, + "REMOVE_CONTAINER": 0.00734094616639478, + "REPLACE": 0.004893964110929853, + "REPLACE_STATEMENT": 0.0032626427406199023, + "REPLACE_SUBTREE": 0.47879282218597063, + "STOP": 0.0065252854812398045 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + }, + "fixture_06": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 720, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 33, + "REMOVE": 4, + "REMOVE_CONTAINER": 14, + "REPLACE": 5, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 1019, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 332, + "invalid_result": 38, + "leaf_container_mismatch": 2, + "not_bindable": 396, + "not_canonical_subtree": 864, + "not_container": 12, + "not_leaf_component": 108, + "not_removable": 3, + "parent_or_comp_precondition": 32, + "target_not_canonical": 4 + }, + "visited_total": 1827, + "dead_total": 1791, + "dead_candidate_rate": 0.9802955665024631, + "visited_shares": { + "ADD": 0.0005473453749315818, + "BIND_PLACEHOLDER": 0.39408866995073893, + "INSERT_STATEMENT": 0.009852216748768473, + "INSERT_SUBTREE": 0.0180623973727422, + "REMOVE": 0.0021893814997263274, + "REMOVE_CONTAINER": 0.007662835249042145, + "REPLACE": 0.002736726874657909, + "REPLACE_STATEMENT": 0.0021893814997263274, + "REPLACE_SUBTREE": 0.5577449370552818, + "STOP": 0.0049261083743842365 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_07": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 40, + "REMOVE": 4, + "REMOVE_CONTAINER": 10, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 10, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 42, + "target_not_canonical": 4 + }, + "visited_total": 1219, + "dead_total": 1187, + "dead_candidate_rate": 0.9737489745693191, + "visited_shares": { + "ADD": 0.003281378178835111, + "BIND_PLACEHOLDER": 0.44298605414273995, + "INSERT_STATEMENT": 0.013125512715340444, + "INSERT_SUBTREE": 0.03281378178835111, + "REMOVE": 0.003281378178835111, + "REMOVE_CONTAINER": 0.008203445447087777, + "REPLACE": 0.004922067268252666, + "REPLACE_STATEMENT": 0.003281378178835111, + "REPLACE_SUBTREE": 0.4815422477440525, + "STOP": 0.006562756357670222 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + } + } + }, + "calibration": { + "per_action": { + "STOP": { + "target_share": 0.18783068783068782, + "visited_share": 0.005201793721973094, + "gap": -0.18262889410871472 + }, + "REPLACE": { + "target_share": 0.1111111111111111, + "visited_share": 0.006547085201793722, + "gap": -0.10456402590931738 + }, + "ADD": { + "target_share": 0.17724867724867724, + "visited_share": 0.0009865470852017937, + "gap": -0.17626213016347544 + }, + "REMOVE": { + "target_share": 0.18253968253968253, + "visited_share": 0.0038565022421524664, + "gap": -0.17868318029753005 + }, + "ADD_CONTAINER": { + "target_share": 0.013227513227513227, + "visited_share": 0.0, + "gap": -0.013227513227513227 + }, + "REMOVE_CONTAINER": { + "target_share": 0.18253968253968253, + "visited_share": 0.008520179372197309, + "gap": -0.1740195031674852 + }, + "INSERT_SUBTREE": { + "target_share": 0.018518518518518517, + "visited_share": 0.012376681614349776, + "gap": -0.006141836904168741 + }, + "REPLACE_SUBTREE": { + "target_share": 0.05026455026455026, + "visited_share": 0.5485201793721973, + "gap": 0.4982556291076471 + }, + "INSERT_STATEMENT": { + "target_share": 0.0, + "visited_share": 0.010403587443946188, + "gap": 0.010403587443946188 + }, + "REPLACE_STATEMENT": { + "target_share": 0.018518518518518517, + "visited_share": 0.003228699551569507, + "gap": -0.01528981896694901 + }, + "BIND_PLACEHOLDER": { + "target_share": 0.0582010582010582, + "visited_share": 0.4003587443946188, + "gap": 0.34215768619356063 + } + }, + "mean_abs_deviation": 0.1546939823173007 + }, + "dead_candidate_rate": 0.9791928251121076, + "applicable_add_recall": 0.10344827586206896, + "verifier_calls": 11092, + "stop_budget_consumed": 58, + "stop_proposals_visited": 58, + "valid_final_rate": 1.0, + "n_records": 8, + "training_metrics": { + "action": 2.143547534942627, + "stmt": 2.8126392364501953, + "slot": 2.276818037033081, + "value": 0.21380357444286346, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + }, + "cell_b": { + "levers": { + "corruption": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + }, + "stop": "legacy" + }, + "metrics": { + "training_targets": { + "samples": 400, + "n_targets": 397, + "counts": { + "STOP": 79, + "REPLACE": 59, + "ADD": 130, + "REMOVE": 44, + "ADD_CONTAINER": 7, + "REMOVE_CONTAINER": 41, + "INSERT_SUBTREE": 12, + "REPLACE_SUBTREE": 9, + "INSERT_STATEMENT": 0, + "REPLACE_STATEMENT": 3, + "BIND_PLACEHOLDER": 13 + }, + "shares": { + "STOP": 0.19899244332493704, + "REPLACE": 0.1486146095717884, + "ADD": 0.327455919395466, + "REMOVE": 0.11083123425692695, + "ADD_CONTAINER": 0.017632241813602016, + "REMOVE_CONTAINER": 0.10327455919395466, + "INSERT_SUBTREE": 0.030226700251889168, + "REPLACE_SUBTREE": 0.022670025188916875, + "INSERT_STATEMENT": 0.0, + "REPLACE_STATEMENT": 0.007556675062972292, + "BIND_PLACEHOLDER": 0.0327455919395466 + }, + "inverse_action_weights": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + } + }, + "decode_demand": { + "overall": { + "visited": { + "ADD": 16, + "BIND_PLACEHOLDER": 5917, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 36, + "REMOVE": 84, + "REMOVE_CONTAINER": 150, + "REPLACE": 8, + "REPLACE_STATEMENT": 126, + "REPLACE_SUBTREE": 8144, + "STOP": 74 + }, + "applicable": { + "ADD": 12, + "BIND_PLACEHOLDER": 1453, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 22, + "REMOVE_CONTAINER": 20, + "REPLACE": 2, + "REPLACE_SUBTREE": 944, + "STOP": 74 + }, + "selected": { + "ADD": 8, + "BIND_PLACEHOLDER": 13, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 12, + "REMOVE_CONTAINER": 4, + "REPLACE": 2, + "REPLACE_SUBTREE": 31, + "STOP": 73 + }, + "rejection_reasons": { + "duplicate_state": 2384, + "invalid_result": 288, + "leaf_container_mismatch": 6, + "not_bindable": 4464, + "not_canonical_subtree": 6912, + "not_container": 130, + "not_removable": 62, + "parent_or_comp_precondition": 36, + "target_not_canonical": 126 + }, + "visited_total": 14703, + "dead_total": 14408, + "dead_candidate_rate": 0.979936067469224, + "visited_shares": { + "ADD": 0.0010882132898048017, + "BIND_PLACEHOLDER": 0.40243487723593824, + "INSERT_STATEMENT": 0.010065972930694417, + "INSERT_SUBTREE": 0.002448479902060804, + "REMOVE": 0.005713119771475209, + "REMOVE_CONTAINER": 0.010201999591920016, + "REPLACE": 0.0005441066449024008, + "REPLACE_STATEMENT": 0.008569679657212814, + "REPLACE_SUBTREE": 0.5539005645106441, + "STOP": 0.005032986465347208 + }, + "verifier_calls": 14629, + "states_with_applicable_add": 74, + "states_recalled": 12, + "applicable_add_recall": 0.16216216216216217 + }, + "by_suite": { + "train": { + "visited": { + "ADD": 16, + "BIND_PLACEHOLDER": 5917, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 36, + "REMOVE": 84, + "REMOVE_CONTAINER": 150, + "REPLACE": 8, + "REPLACE_STATEMENT": 126, + "REPLACE_SUBTREE": 8144, + "STOP": 74 + }, + "applicable": { + "ADD": 12, + "BIND_PLACEHOLDER": 1453, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 22, + "REMOVE_CONTAINER": 20, + "REPLACE": 2, + "REPLACE_SUBTREE": 944, + "STOP": 74 + }, + "selected": { + "ADD": 8, + "BIND_PLACEHOLDER": 13, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 12, + "REMOVE_CONTAINER": 4, + "REPLACE": 2, + "REPLACE_SUBTREE": 31, + "STOP": 73 + }, + "rejection_reasons": { + "duplicate_state": 2384, + "invalid_result": 288, + "leaf_container_mismatch": 6, + "not_bindable": 4464, + "not_canonical_subtree": 6912, + "not_container": 130, + "not_removable": 62, + "parent_or_comp_precondition": 36, + "target_not_canonical": 126 + }, + "visited_total": 14703, + "dead_total": 14408, + "dead_candidate_rate": 0.979936067469224, + "visited_shares": { + "ADD": 0.0010882132898048017, + "BIND_PLACEHOLDER": 0.40243487723593824, + "INSERT_STATEMENT": 0.010065972930694417, + "INSERT_SUBTREE": 0.002448479902060804, + "REMOVE": 0.005713119771475209, + "REMOVE_CONTAINER": 0.010201999591920016, + "REPLACE": 0.0005441066449024008, + "REPLACE_STATEMENT": 0.008569679657212814, + "REPLACE_SUBTREE": 0.5539005645106441, + "STOP": 0.005032986465347208 + }, + "verifier_calls": 0, + "states_with_applicable_add": 74, + "states_recalled": 12, + "applicable_add_recall": 0.16216216216216217 + } + }, + "by_source": { + "fixture_00": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 688, + "INSERT_STATEMENT": 16, + "REMOVE": 13, + "REMOVE_CONTAINER": 17, + "REPLACE": 1, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 293, + "STOP": 8 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 112, + "INSERT_STATEMENT": 16, + "REMOVE": 5, + "REPLACE_SUBTREE": 5, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 4, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE_SUBTREE": 2, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 115, + "leaf_container_mismatch": 1, + "not_bindable": 576, + "not_canonical_subtree": 288, + "not_container": 17, + "not_removable": 8, + "target_not_canonical": 16 + }, + "visited_total": 1053, + "dead_total": 1021, + "dead_candidate_rate": 0.9696106362773029, + "visited_shares": { + "ADD": 0.000949667616334283, + "BIND_PLACEHOLDER": 0.6533713200379867, + "INSERT_STATEMENT": 0.015194681861348529, + "REMOVE": 0.012345679012345678, + "REMOVE_CONTAINER": 0.016144349477682812, + "REPLACE": 0.000949667616334283, + "REPLACE_STATEMENT": 0.015194681861348529, + "REPLACE_SUBTREE": 0.27825261158594494, + "STOP": 0.007597340930674264 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 1, + "applicable_add_recall": 0.125 + }, + "fixture_01": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 21, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 16, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2556, + "dead_total": 2512, + "dead_candidate_rate": 0.9827856025039123, + "visited_shares": { + "ADD": 0.0011737089201877935, + "BIND_PLACEHOLDER": 0.3380281690140845, + "INSERT_STATEMENT": 0.008607198748043818, + "INSERT_SUBTREE": 0.0035211267605633804, + "REMOVE": 0.003912363067292645, + "REMOVE_CONTAINER": 0.008215962441314555, + "REPLACE_STATEMENT": 0.006259780907668232, + "REPLACE_SUBTREE": 0.6259780907668232, + "STOP": 0.004303599374021909 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_02": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 544, + "INSERT_STATEMENT": 14, + "REMOVE": 10, + "REMOVE_CONTAINER": 14, + "REPLACE": 3, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 576, + "STOP": 7 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 40, + "INSERT_STATEMENT": 14, + "REMOVE": 3, + "REPLACE": 1, + "STOP": 7 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 4, + "INSERT_STATEMENT": 14, + "REMOVE": 1, + "REPLACE": 1, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 38, + "leaf_container_mismatch": 2, + "not_bindable": 504, + "not_canonical_subtree": 576, + "not_container": 14, + "not_removable": 7, + "target_not_canonical": 16 + }, + "visited_total": 1185, + "dead_total": 1157, + "dead_candidate_rate": 0.9763713080168777, + "visited_shares": { + "ADD": 0.0008438818565400844, + "BIND_PLACEHOLDER": 0.45907172995780593, + "INSERT_STATEMENT": 0.01181434599156118, + "REMOVE": 0.008438818565400843, + "REMOVE_CONTAINER": 0.01181434599156118, + "REPLACE": 0.002531645569620253, + "REPLACE_STATEMENT": 0.01350210970464135, + "REPLACE_SUBTREE": 0.4860759493670886, + "STOP": 0.00590717299578059 + }, + "verifier_calls": 0, + "states_with_applicable_add": 7, + "states_recalled": 1, + "applicable_add_recall": 0.14285714285714285 + }, + "fixture_03": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 579, + "INSERT_STATEMENT": 14, + "REMOVE": 9, + "REMOVE_CONTAINER": 15, + "REPLACE": 3, + "REPLACE_STATEMENT": 14, + "REPLACE_SUBTREE": 578, + "STOP": 7 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 75, + "INSERT_STATEMENT": 14, + "REMOVE": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 2, + "STOP": 7 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 3, + "INSERT_STATEMENT": 14, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 1, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 74, + "leaf_container_mismatch": 2, + "not_bindable": 504, + "not_canonical_subtree": 576, + "not_container": 15, + "not_removable": 7, + "target_not_canonical": 14 + }, + "visited_total": 1220, + "dead_total": 1192, + "dead_candidate_rate": 0.9770491803278688, + "visited_shares": { + "ADD": 0.000819672131147541, + "BIND_PLACEHOLDER": 0.4745901639344262, + "INSERT_STATEMENT": 0.011475409836065573, + "REMOVE": 0.007377049180327869, + "REMOVE_CONTAINER": 0.012295081967213115, + "REPLACE": 0.002459016393442623, + "REPLACE_STATEMENT": 0.011475409836065573, + "REPLACE_SUBTREE": 0.4737704918032787, + "STOP": 0.005737704918032787 + }, + "verifier_calls": 0, + "states_with_applicable_add": 7, + "states_recalled": 1, + "applicable_add_recall": 0.14285714285714285 + }, + "fixture_04": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 650, + "INSERT_STATEMENT": 16, + "REMOVE": 12, + "REMOVE_CONTAINER": 18, + "REPLACE": 1, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 297, + "STOP": 8 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 74, + "INSERT_STATEMENT": 16, + "REMOVE": 4, + "REPLACE_SUBTREE": 9, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 2, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 81, + "leaf_container_mismatch": 1, + "not_bindable": 576, + "not_canonical_subtree": 288, + "not_container": 18, + "not_removable": 8, + "target_not_canonical": 16 + }, + "visited_total": 1019, + "dead_total": 988, + "dead_candidate_rate": 0.9695780176643768, + "visited_shares": { + "ADD": 0.0009813542688910696, + "BIND_PLACEHOLDER": 0.6378802747791953, + "INSERT_STATEMENT": 0.015701668302257114, + "REMOVE": 0.011776251226692836, + "REMOVE_CONTAINER": 0.017664376840039256, + "REPLACE": 0.0009813542688910696, + "REPLACE_STATEMENT": 0.015701668302257114, + "REPLACE_SUBTREE": 0.2914622178606477, + "STOP": 0.007850834151128557 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 1, + "applicable_add_recall": 0.125 + }, + "fixture_05": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 22, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 17, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2557, + "dead_total": 2513, + "dead_candidate_rate": 0.9827923347673054, + "visited_shares": { + "ADD": 0.0011732499022291747, + "BIND_PLACEHOLDER": 0.3378959718420024, + "INSERT_STATEMENT": 0.008603832616347283, + "INSERT_SUBTREE": 0.0035197497066875244, + "REMOVE": 0.003910833007430583, + "REMOVE_CONTAINER": 0.008603832616347283, + "REPLACE_STATEMENT": 0.006257332811888932, + "REPLACE_SUBTREE": 0.6257332811888933, + "STOP": 0.004301916308173641 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_06": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 22, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 17, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2557, + "dead_total": 2513, + "dead_candidate_rate": 0.9827923347673054, + "visited_shares": { + "ADD": 0.0011732499022291747, + "BIND_PLACEHOLDER": 0.3378959718420024, + "INSERT_STATEMENT": 0.008603832616347283, + "INSERT_SUBTREE": 0.0035197497066875244, + "REMOVE": 0.003910833007430583, + "REMOVE_CONTAINER": 0.008603832616347283, + "REPLACE_STATEMENT": 0.006257332811888932, + "REPLACE_SUBTREE": 0.6257332811888933, + "STOP": 0.004301916308173641 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_07": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 21, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 16, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2556, + "dead_total": 2512, + "dead_candidate_rate": 0.9827856025039123, + "visited_shares": { + "ADD": 0.0011737089201877935, + "BIND_PLACEHOLDER": 0.3380281690140845, + "INSERT_STATEMENT": 0.008607198748043818, + "INSERT_SUBTREE": 0.0035211267605633804, + "REMOVE": 0.003912363067292645, + "REMOVE_CONTAINER": 0.008215962441314555, + "REPLACE_STATEMENT": 0.006259780907668232, + "REPLACE_SUBTREE": 0.6259780907668232, + "STOP": 0.004303599374021909 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + } + } + }, + "calibration": { + "per_action": { + "STOP": { + "target_share": 0.19899244332493704, + "visited_share": 0.005032986465347208, + "gap": -0.19395945685958982 + }, + "REPLACE": { + "target_share": 0.1486146095717884, + "visited_share": 0.0005441066449024008, + "gap": -0.148070502926886 + }, + "ADD": { + "target_share": 0.327455919395466, + "visited_share": 0.0010882132898048017, + "gap": -0.3263677061056612 + }, + "REMOVE": { + "target_share": 0.11083123425692695, + "visited_share": 0.005713119771475209, + "gap": -0.10511811448545175 + }, + "ADD_CONTAINER": { + "target_share": 0.017632241813602016, + "visited_share": 0.0, + "gap": -0.017632241813602016 + }, + "REMOVE_CONTAINER": { + "target_share": 0.10327455919395466, + "visited_share": 0.010201999591920016, + "gap": -0.09307255960203464 + }, + "INSERT_SUBTREE": { + "target_share": 0.030226700251889168, + "visited_share": 0.002448479902060804, + "gap": -0.027778220349828363 + }, + "REPLACE_SUBTREE": { + "target_share": 0.022670025188916875, + "visited_share": 0.5539005645106441, + "gap": 0.5312305393217271 + }, + "INSERT_STATEMENT": { + "target_share": 0.0, + "visited_share": 0.010065972930694417, + "gap": 0.010065972930694417 + }, + "REPLACE_STATEMENT": { + "target_share": 0.007556675062972292, + "visited_share": 0.008569679657212814, + "gap": 0.0010130045942405219 + }, + "BIND_PLACEHOLDER": { + "target_share": 0.0327455919395466, + "visited_share": 0.40243487723593824, + "gap": 0.36968928529639167 + } + }, + "mean_abs_deviation": 0.16581796402600976 + }, + "dead_candidate_rate": 0.979936067469224, + "applicable_add_recall": 0.16216216216216217, + "verifier_calls": 14629, + "stop_budget_consumed": 74, + "stop_proposals_visited": 74, + "valid_final_rate": 1.0, + "n_records": 8, + "training_metrics": { + "action": 2.112842082977295, + "stmt": 2.703423023223877, + "comp": 2.7941222190856934, + "slot": 2.7572789192199707, + "value": 0.16673196852207184, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + }, + "cell_c": { + "levers": { + "corruption": null, + "stop": "corrected" + }, + "metrics": { + "training_targets": { + "samples": 400, + "n_targets": 378, + "counts": { + "STOP": 71, + "REPLACE": 42, + "ADD": 67, + "REMOVE": 69, + "ADD_CONTAINER": 5, + "REMOVE_CONTAINER": 69, + "INSERT_SUBTREE": 7, + "REPLACE_SUBTREE": 19, + "INSERT_STATEMENT": 0, + "REPLACE_STATEMENT": 7, + "BIND_PLACEHOLDER": 22 + }, + "shares": { + "STOP": 0.18783068783068782, + "REPLACE": 0.1111111111111111, + "ADD": 0.17724867724867724, + "REMOVE": 0.18253968253968253, + "ADD_CONTAINER": 0.013227513227513227, + "REMOVE_CONTAINER": 0.18253968253968253, + "INSERT_SUBTREE": 0.018518518518518517, + "REPLACE_SUBTREE": 0.05026455026455026, + "INSERT_STATEMENT": 0.0, + "REPLACE_STATEMENT": 0.018518518518518517, + "BIND_PLACEHOLDER": 0.0582010582010582 + }, + "inverse_action_weights": null + }, + "decode_demand": { + "overall": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 11092, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + }, + "by_suite": { + "train": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 0, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + } + }, + "by_source": { + "fixture_00": { + "visited": { + "BIND_PLACEHOLDER": 576, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 15, + "REPLACE": 3, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 297, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 9, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 185, + "invalid_result": 1, + "leaf_container_mismatch": 1, + "not_bindable": 252, + "not_canonical_subtree": 288, + "not_container": 15, + "not_leaf_component": 144, + "not_removable": 6, + "target_not_canonical": 2 + }, + "visited_total": 918, + "dead_total": 894, + "dead_candidate_rate": 0.9738562091503268, + "visited_shares": { + "BIND_PLACEHOLDER": 0.6274509803921569, + "INSERT_STATEMENT": 0.013071895424836602, + "REMOVE": 0.007625272331154684, + "REMOVE_CONTAINER": 0.016339869281045753, + "REPLACE": 0.0032679738562091504, + "REPLACE_STATEMENT": 0.002178649237472767, + "REPLACE_SUBTREE": 0.3235294117647059, + "STOP": 0.006535947712418301 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_01": { + "visited": { + "ADD": 2, + "BIND_PLACEHOLDER": 648, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 17, + "REMOVE": 3, + "REMOVE_CONTAINER": 14, + "REPLACE": 2, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 1022, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 3, + "REPLACE_SUBTREE": 230, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 17, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 7, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 407, + "invalid_result": 73, + "leaf_container_mismatch": 1, + "not_bindable": 432, + "not_canonical_subtree": 720, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 2, + "parent_or_comp_precondition": 17, + "target_not_canonical": 2 + }, + "visited_total": 1737, + "dead_total": 1701, + "dead_candidate_rate": 0.9792746113989638, + "visited_shares": { + "ADD": 0.0011514104778353484, + "BIND_PLACEHOLDER": 0.37305699481865284, + "INSERT_STATEMENT": 0.010362694300518135, + "INSERT_SUBTREE": 0.00978698906160046, + "REMOVE": 0.0017271157167530224, + "REMOVE_CONTAINER": 0.008059873344847437, + "REPLACE": 0.0011514104778353484, + "REPLACE_STATEMENT": 0.0011514104778353484, + "REPLACE_SUBTREE": 0.588370754173863, + "STOP": 0.0051813471502590676 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_02": { + "visited": { + "BIND_PLACEHOLDER": 504, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1431, + "dead_total": 1407, + "dead_candidate_rate": 0.9832285115303984, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3522012578616352, + "INSERT_STATEMENT": 0.008385744234800839, + "REMOVE": 0.004891684136967156, + "REMOVE_CONTAINER": 0.007686932215234102, + "REPLACE": 0.011879804332634521, + "REPLACE_STATEMENT": 0.0041928721174004195, + "REPLACE_SUBTREE": 0.6065688329839273, + "STOP": 0.0041928721174004195 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_03": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 8, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 72, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 75, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 8 + }, + "visited_total": 1397, + "dead_total": 1373, + "dead_candidate_rate": 0.9828203292770222, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3350035790980673, + "INSERT_STATEMENT": 0.008589835361488905, + "REMOVE": 0.005010737294201861, + "REMOVE_CONTAINER": 0.007874015748031496, + "REPLACE": 0.012168933428775949, + "REPLACE_STATEMENT": 0.00572655690765927, + "REPLACE_SUBTREE": 0.6213314244810307, + "STOP": 0.004294917680744453 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_04": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1395, + "dead_total": 1371, + "dead_candidate_rate": 0.9827956989247312, + "visited_shares": { + "BIND_PLACEHOLDER": 0.33548387096774196, + "INSERT_STATEMENT": 0.008602150537634409, + "REMOVE": 0.005017921146953405, + "REMOVE_CONTAINER": 0.007885304659498209, + "REPLACE": 0.012186379928315413, + "REPLACE_STATEMENT": 0.004301075268817204, + "REPLACE_SUBTREE": 0.6222222222222222, + "STOP": 0.004301075268817204 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_05": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 48, + "REMOVE": 4, + "REMOVE_CONTAINER": 9, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 9, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 50, + "target_not_canonical": 4 + }, + "visited_total": 1226, + "dead_total": 1194, + "dead_candidate_rate": 0.9738988580750407, + "visited_shares": { + "ADD": 0.0032626427406199023, + "BIND_PLACEHOLDER": 0.44045676998368677, + "INSERT_STATEMENT": 0.013050570962479609, + "INSERT_SUBTREE": 0.03915171288743882, + "REMOVE": 0.0032626427406199023, + "REMOVE_CONTAINER": 0.00734094616639478, + "REPLACE": 0.004893964110929853, + "REPLACE_STATEMENT": 0.0032626427406199023, + "REPLACE_SUBTREE": 0.47879282218597063, + "STOP": 0.0065252854812398045 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + }, + "fixture_06": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 720, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 33, + "REMOVE": 4, + "REMOVE_CONTAINER": 14, + "REPLACE": 5, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 1019, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 332, + "invalid_result": 38, + "leaf_container_mismatch": 2, + "not_bindable": 396, + "not_canonical_subtree": 864, + "not_container": 12, + "not_leaf_component": 108, + "not_removable": 3, + "parent_or_comp_precondition": 32, + "target_not_canonical": 4 + }, + "visited_total": 1827, + "dead_total": 1791, + "dead_candidate_rate": 0.9802955665024631, + "visited_shares": { + "ADD": 0.0005473453749315818, + "BIND_PLACEHOLDER": 0.39408866995073893, + "INSERT_STATEMENT": 0.009852216748768473, + "INSERT_SUBTREE": 0.0180623973727422, + "REMOVE": 0.0021893814997263274, + "REMOVE_CONTAINER": 0.007662835249042145, + "REPLACE": 0.002736726874657909, + "REPLACE_STATEMENT": 0.0021893814997263274, + "REPLACE_SUBTREE": 0.5577449370552818, + "STOP": 0.0049261083743842365 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_07": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 40, + "REMOVE": 4, + "REMOVE_CONTAINER": 10, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 10, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 42, + "target_not_canonical": 4 + }, + "visited_total": 1219, + "dead_total": 1187, + "dead_candidate_rate": 0.9737489745693191, + "visited_shares": { + "ADD": 0.003281378178835111, + "BIND_PLACEHOLDER": 0.44298605414273995, + "INSERT_STATEMENT": 0.013125512715340444, + "INSERT_SUBTREE": 0.03281378178835111, + "REMOVE": 0.003281378178835111, + "REMOVE_CONTAINER": 0.008203445447087777, + "REPLACE": 0.004922067268252666, + "REPLACE_STATEMENT": 0.003281378178835111, + "REPLACE_SUBTREE": 0.4815422477440525, + "STOP": 0.006562756357670222 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + } + } + }, + "calibration": { + "per_action": { + "STOP": { + "target_share": 0.18783068783068782, + "visited_share": 0.005201793721973094, + "gap": -0.18262889410871472 + }, + "REPLACE": { + "target_share": 0.1111111111111111, + "visited_share": 0.006547085201793722, + "gap": -0.10456402590931738 + }, + "ADD": { + "target_share": 0.17724867724867724, + "visited_share": 0.0009865470852017937, + "gap": -0.17626213016347544 + }, + "REMOVE": { + "target_share": 0.18253968253968253, + "visited_share": 0.0038565022421524664, + "gap": -0.17868318029753005 + }, + "ADD_CONTAINER": { + "target_share": 0.013227513227513227, + "visited_share": 0.0, + "gap": -0.013227513227513227 + }, + "REMOVE_CONTAINER": { + "target_share": 0.18253968253968253, + "visited_share": 0.008520179372197309, + "gap": -0.1740195031674852 + }, + "INSERT_SUBTREE": { + "target_share": 0.018518518518518517, + "visited_share": 0.012376681614349776, + "gap": -0.006141836904168741 + }, + "REPLACE_SUBTREE": { + "target_share": 0.05026455026455026, + "visited_share": 0.5485201793721973, + "gap": 0.4982556291076471 + }, + "INSERT_STATEMENT": { + "target_share": 0.0, + "visited_share": 0.010403587443946188, + "gap": 0.010403587443946188 + }, + "REPLACE_STATEMENT": { + "target_share": 0.018518518518518517, + "visited_share": 0.003228699551569507, + "gap": -0.01528981896694901 + }, + "BIND_PLACEHOLDER": { + "target_share": 0.0582010582010582, + "visited_share": 0.4003587443946188, + "gap": 0.34215768619356063 + } + }, + "mean_abs_deviation": 0.1546939823173007 + }, + "dead_candidate_rate": 0.9791928251121076, + "applicable_add_recall": 0.10344827586206896, + "verifier_calls": 11092, + "stop_budget_consumed": 58, + "stop_proposals_visited": 58, + "valid_final_rate": 1.0, + "n_records": 8, + "training_metrics": { + "action": 2.143547534942627, + "stmt": 2.8126392364501953, + "slot": 2.276818037033081, + "value": 0.21380357444286346, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + }, + "cell_d": { + "levers": { + "corruption": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + }, + "stop": "corrected" + }, + "metrics": { + "training_targets": { + "samples": 400, + "n_targets": 397, + "counts": { + "STOP": 79, + "REPLACE": 59, + "ADD": 130, + "REMOVE": 44, + "ADD_CONTAINER": 7, + "REMOVE_CONTAINER": 41, + "INSERT_SUBTREE": 12, + "REPLACE_SUBTREE": 9, + "INSERT_STATEMENT": 0, + "REPLACE_STATEMENT": 3, + "BIND_PLACEHOLDER": 13 + }, + "shares": { + "STOP": 0.19899244332493704, + "REPLACE": 0.1486146095717884, + "ADD": 0.327455919395466, + "REMOVE": 0.11083123425692695, + "ADD_CONTAINER": 0.017632241813602016, + "REMOVE_CONTAINER": 0.10327455919395466, + "INSERT_SUBTREE": 0.030226700251889168, + "REPLACE_SUBTREE": 0.022670025188916875, + "INSERT_STATEMENT": 0.0, + "REPLACE_STATEMENT": 0.007556675062972292, + "BIND_PLACEHOLDER": 0.0327455919395466 + }, + "inverse_action_weights": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + } + }, + "decode_demand": { + "overall": { + "visited": { + "ADD": 16, + "BIND_PLACEHOLDER": 5917, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 36, + "REMOVE": 84, + "REMOVE_CONTAINER": 150, + "REPLACE": 8, + "REPLACE_STATEMENT": 126, + "REPLACE_SUBTREE": 8145, + "STOP": 74 + }, + "applicable": { + "ADD": 12, + "BIND_PLACEHOLDER": 1453, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 22, + "REMOVE_CONTAINER": 20, + "REPLACE": 2, + "REPLACE_SUBTREE": 945, + "STOP": 74 + }, + "selected": { + "ADD": 8, + "BIND_PLACEHOLDER": 13, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 12, + "REMOVE_CONTAINER": 4, + "REPLACE": 2, + "REPLACE_SUBTREE": 32, + "STOP": 73 + }, + "rejection_reasons": { + "duplicate_state": 2384, + "invalid_result": 288, + "leaf_container_mismatch": 6, + "not_bindable": 4464, + "not_canonical_subtree": 6912, + "not_container": 130, + "not_removable": 62, + "parent_or_comp_precondition": 36, + "target_not_canonical": 126 + }, + "visited_total": 14704, + "dead_total": 14408, + "dead_candidate_rate": 0.9798694232861807, + "visited_shares": { + "ADD": 0.001088139281828074, + "BIND_PLACEHOLDER": 0.4024075081610446, + "INSERT_STATEMENT": 0.010065288356909684, + "INSERT_SUBTREE": 0.0024483133841131663, + "REMOVE": 0.005712731229597388, + "REMOVE_CONTAINER": 0.010201305767138194, + "REPLACE": 0.000544069640914037, + "REPLACE_STATEMENT": 0.008569096844396083, + "REPLACE_SUBTREE": 0.5539309031556039, + "STOP": 0.005032644178454842 + }, + "verifier_calls": 14630, + "states_with_applicable_add": 74, + "states_recalled": 12, + "applicable_add_recall": 0.16216216216216217 + }, + "by_suite": { + "train": { + "visited": { + "ADD": 16, + "BIND_PLACEHOLDER": 5917, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 36, + "REMOVE": 84, + "REMOVE_CONTAINER": 150, + "REPLACE": 8, + "REPLACE_STATEMENT": 126, + "REPLACE_SUBTREE": 8145, + "STOP": 74 + }, + "applicable": { + "ADD": 12, + "BIND_PLACEHOLDER": 1453, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 22, + "REMOVE_CONTAINER": 20, + "REPLACE": 2, + "REPLACE_SUBTREE": 945, + "STOP": 74 + }, + "selected": { + "ADD": 8, + "BIND_PLACEHOLDER": 13, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 12, + "REMOVE_CONTAINER": 4, + "REPLACE": 2, + "REPLACE_SUBTREE": 32, + "STOP": 73 + }, + "rejection_reasons": { + "duplicate_state": 2384, + "invalid_result": 288, + "leaf_container_mismatch": 6, + "not_bindable": 4464, + "not_canonical_subtree": 6912, + "not_container": 130, + "not_removable": 62, + "parent_or_comp_precondition": 36, + "target_not_canonical": 126 + }, + "visited_total": 14704, + "dead_total": 14408, + "dead_candidate_rate": 0.9798694232861807, + "visited_shares": { + "ADD": 0.001088139281828074, + "BIND_PLACEHOLDER": 0.4024075081610446, + "INSERT_STATEMENT": 0.010065288356909684, + "INSERT_SUBTREE": 0.0024483133841131663, + "REMOVE": 0.005712731229597388, + "REMOVE_CONTAINER": 0.010201305767138194, + "REPLACE": 0.000544069640914037, + "REPLACE_STATEMENT": 0.008569096844396083, + "REPLACE_SUBTREE": 0.5539309031556039, + "STOP": 0.005032644178454842 + }, + "verifier_calls": 0, + "states_with_applicable_add": 74, + "states_recalled": 12, + "applicable_add_recall": 0.16216216216216217 + } + }, + "by_source": { + "fixture_00": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 688, + "INSERT_STATEMENT": 16, + "REMOVE": 13, + "REMOVE_CONTAINER": 17, + "REPLACE": 1, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 293, + "STOP": 8 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 112, + "INSERT_STATEMENT": 16, + "REMOVE": 5, + "REPLACE_SUBTREE": 5, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 4, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE_SUBTREE": 2, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 115, + "leaf_container_mismatch": 1, + "not_bindable": 576, + "not_canonical_subtree": 288, + "not_container": 17, + "not_removable": 8, + "target_not_canonical": 16 + }, + "visited_total": 1053, + "dead_total": 1021, + "dead_candidate_rate": 0.9696106362773029, + "visited_shares": { + "ADD": 0.000949667616334283, + "BIND_PLACEHOLDER": 0.6533713200379867, + "INSERT_STATEMENT": 0.015194681861348529, + "REMOVE": 0.012345679012345678, + "REMOVE_CONTAINER": 0.016144349477682812, + "REPLACE": 0.000949667616334283, + "REPLACE_STATEMENT": 0.015194681861348529, + "REPLACE_SUBTREE": 0.27825261158594494, + "STOP": 0.007597340930674264 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 1, + "applicable_add_recall": 0.125 + }, + "fixture_01": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 21, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 16, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2556, + "dead_total": 2512, + "dead_candidate_rate": 0.9827856025039123, + "visited_shares": { + "ADD": 0.0011737089201877935, + "BIND_PLACEHOLDER": 0.3380281690140845, + "INSERT_STATEMENT": 0.008607198748043818, + "INSERT_SUBTREE": 0.0035211267605633804, + "REMOVE": 0.003912363067292645, + "REMOVE_CONTAINER": 0.008215962441314555, + "REPLACE_STATEMENT": 0.006259780907668232, + "REPLACE_SUBTREE": 0.6259780907668232, + "STOP": 0.004303599374021909 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_02": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 544, + "INSERT_STATEMENT": 14, + "REMOVE": 10, + "REMOVE_CONTAINER": 14, + "REPLACE": 3, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 576, + "STOP": 7 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 40, + "INSERT_STATEMENT": 14, + "REMOVE": 3, + "REPLACE": 1, + "STOP": 7 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 4, + "INSERT_STATEMENT": 14, + "REMOVE": 1, + "REPLACE": 1, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 38, + "leaf_container_mismatch": 2, + "not_bindable": 504, + "not_canonical_subtree": 576, + "not_container": 14, + "not_removable": 7, + "target_not_canonical": 16 + }, + "visited_total": 1185, + "dead_total": 1157, + "dead_candidate_rate": 0.9763713080168777, + "visited_shares": { + "ADD": 0.0008438818565400844, + "BIND_PLACEHOLDER": 0.45907172995780593, + "INSERT_STATEMENT": 0.01181434599156118, + "REMOVE": 0.008438818565400843, + "REMOVE_CONTAINER": 0.01181434599156118, + "REPLACE": 0.002531645569620253, + "REPLACE_STATEMENT": 0.01350210970464135, + "REPLACE_SUBTREE": 0.4860759493670886, + "STOP": 0.00590717299578059 + }, + "verifier_calls": 0, + "states_with_applicable_add": 7, + "states_recalled": 1, + "applicable_add_recall": 0.14285714285714285 + }, + "fixture_03": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 579, + "INSERT_STATEMENT": 14, + "REMOVE": 9, + "REMOVE_CONTAINER": 15, + "REPLACE": 3, + "REPLACE_STATEMENT": 14, + "REPLACE_SUBTREE": 578, + "STOP": 7 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 75, + "INSERT_STATEMENT": 14, + "REMOVE": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 2, + "STOP": 7 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 3, + "INSERT_STATEMENT": 14, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 1, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 74, + "leaf_container_mismatch": 2, + "not_bindable": 504, + "not_canonical_subtree": 576, + "not_container": 15, + "not_removable": 7, + "target_not_canonical": 14 + }, + "visited_total": 1220, + "dead_total": 1192, + "dead_candidate_rate": 0.9770491803278688, + "visited_shares": { + "ADD": 0.000819672131147541, + "BIND_PLACEHOLDER": 0.4745901639344262, + "INSERT_STATEMENT": 0.011475409836065573, + "REMOVE": 0.007377049180327869, + "REMOVE_CONTAINER": 0.012295081967213115, + "REPLACE": 0.002459016393442623, + "REPLACE_STATEMENT": 0.011475409836065573, + "REPLACE_SUBTREE": 0.4737704918032787, + "STOP": 0.005737704918032787 + }, + "verifier_calls": 0, + "states_with_applicable_add": 7, + "states_recalled": 1, + "applicable_add_recall": 0.14285714285714285 + }, + "fixture_04": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 650, + "INSERT_STATEMENT": 16, + "REMOVE": 12, + "REMOVE_CONTAINER": 18, + "REPLACE": 1, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 298, + "STOP": 8 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 74, + "INSERT_STATEMENT": 16, + "REMOVE": 4, + "REPLACE_SUBTREE": 10, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 2, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE_SUBTREE": 5, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 81, + "leaf_container_mismatch": 1, + "not_bindable": 576, + "not_canonical_subtree": 288, + "not_container": 18, + "not_removable": 8, + "target_not_canonical": 16 + }, + "visited_total": 1020, + "dead_total": 988, + "dead_candidate_rate": 0.9686274509803922, + "visited_shares": { + "ADD": 0.000980392156862745, + "BIND_PLACEHOLDER": 0.6372549019607843, + "INSERT_STATEMENT": 0.01568627450980392, + "REMOVE": 0.011764705882352941, + "REMOVE_CONTAINER": 0.01764705882352941, + "REPLACE": 0.000980392156862745, + "REPLACE_STATEMENT": 0.01568627450980392, + "REPLACE_SUBTREE": 0.29215686274509806, + "STOP": 0.00784313725490196 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 1, + "applicable_add_recall": 0.125 + }, + "fixture_05": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 22, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 17, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2557, + "dead_total": 2513, + "dead_candidate_rate": 0.9827923347673054, + "visited_shares": { + "ADD": 0.0011732499022291747, + "BIND_PLACEHOLDER": 0.3378959718420024, + "INSERT_STATEMENT": 0.008603832616347283, + "INSERT_SUBTREE": 0.0035197497066875244, + "REMOVE": 0.003910833007430583, + "REMOVE_CONTAINER": 0.008603832616347283, + "REPLACE_STATEMENT": 0.006257332811888932, + "REPLACE_SUBTREE": 0.6257332811888933, + "STOP": 0.004301916308173641 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_06": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 22, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 17, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2557, + "dead_total": 2513, + "dead_candidate_rate": 0.9827923347673054, + "visited_shares": { + "ADD": 0.0011732499022291747, + "BIND_PLACEHOLDER": 0.3378959718420024, + "INSERT_STATEMENT": 0.008603832616347283, + "INSERT_SUBTREE": 0.0035197497066875244, + "REMOVE": 0.003910833007430583, + "REMOVE_CONTAINER": 0.008603832616347283, + "REPLACE_STATEMENT": 0.006257332811888932, + "REPLACE_SUBTREE": 0.6257332811888933, + "STOP": 0.004301916308173641 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_07": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 21, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 16, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2556, + "dead_total": 2512, + "dead_candidate_rate": 0.9827856025039123, + "visited_shares": { + "ADD": 0.0011737089201877935, + "BIND_PLACEHOLDER": 0.3380281690140845, + "INSERT_STATEMENT": 0.008607198748043818, + "INSERT_SUBTREE": 0.0035211267605633804, + "REMOVE": 0.003912363067292645, + "REMOVE_CONTAINER": 0.008215962441314555, + "REPLACE_STATEMENT": 0.006259780907668232, + "REPLACE_SUBTREE": 0.6259780907668232, + "STOP": 0.004303599374021909 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + } + } + }, + "calibration": { + "per_action": { + "STOP": { + "target_share": 0.19899244332493704, + "visited_share": 0.005032644178454842, + "gap": -0.1939597991464822 + }, + "REPLACE": { + "target_share": 0.1486146095717884, + "visited_share": 0.000544069640914037, + "gap": -0.14807053993087438 + }, + "ADD": { + "target_share": 0.327455919395466, + "visited_share": 0.001088139281828074, + "gap": -0.32636778011363793 + }, + "REMOVE": { + "target_share": 0.11083123425692695, + "visited_share": 0.005712731229597388, + "gap": -0.10511850302732956 + }, + "ADD_CONTAINER": { + "target_share": 0.017632241813602016, + "visited_share": 0.0, + "gap": -0.017632241813602016 + }, + "REMOVE_CONTAINER": { + "target_share": 0.10327455919395466, + "visited_share": 0.010201305767138194, + "gap": -0.09307325342681647 + }, + "INSERT_SUBTREE": { + "target_share": 0.030226700251889168, + "visited_share": 0.0024483133841131663, + "gap": -0.027778386867776002 + }, + "REPLACE_SUBTREE": { + "target_share": 0.022670025188916875, + "visited_share": 0.5539309031556039, + "gap": 0.531260877966687 + }, + "INSERT_STATEMENT": { + "target_share": 0.0, + "visited_share": 0.010065288356909684, + "gap": 0.010065288356909684 + }, + "REPLACE_STATEMENT": { + "target_share": 0.007556675062972292, + "visited_share": 0.008569096844396083, + "gap": 0.0010124217814237906 + }, + "BIND_PLACEHOLDER": { + "target_share": 0.0327455919395466, + "visited_share": 0.4024075081610446, + "gap": 0.369661916221498 + } + }, + "mean_abs_deviation": 0.16581827351391246 + }, + "dead_candidate_rate": 0.9798694232861807, + "applicable_add_recall": 0.16216216216216217, + "verifier_calls": 14630, + "stop_budget_consumed": 73, + "stop_proposals_visited": 74, + "valid_final_rate": 1.0, + "n_records": 8, + "training_metrics": { + "action": 2.112842082977295, + "stmt": 2.703423023223877, + "comp": 2.7941222190856934, + "slot": 2.7572789192199707, + "value": 0.16673196852207184, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + } + }, + "threshold_checks": { + "t1_add_target_share_gains": { + "cell_b_minus_cell_a": 0.15020724214678874, + "cell_d_minus_cell_c": 0.15020724214678874 + }, + "t1_ok": true, + "t2_valid_final": { + "cell_b_vs_cell_a": true, + "cell_d_vs_cell_c": true + }, + "t2_ok": true, + "t3_stop_accounting": { + "cell_c_vs_cell_a": { + "stop_budget_legacy": 58, + "stop_budget_corrected": 58, + "budget_ok": true, + "valid_final_ok": true + }, + "cell_d_vs_cell_b": { + "stop_budget_legacy": 74, + "stop_budget_corrected": 73, + "budget_ok": true, + "valid_final_ok": true + } + }, + "t3_ok": true, + "t4_lever_isolation": { + "b_minus_a": [ + "corruption" + ], + "c_minus_a": [ + "stop" + ], + "d_minus_b": [ + "stop" + ], + "d_minus_c": [ + "corruption" + ] + }, + "t4_ok": true + }, + "verdicts": { + "corruption_add_balanced": "adopted", + "stop_slot_corrected": "adopted" + }, + "baseline_audit": { + "training_targets": { + "samples": 400, + "n_targets": 378, + "counts": { + "STOP": 71, + "REPLACE": 42, + "ADD": 67, + "REMOVE": 69, + "ADD_CONTAINER": 5, + "REMOVE_CONTAINER": 69, + "INSERT_SUBTREE": 7, + "REPLACE_SUBTREE": 19, + "INSERT_STATEMENT": 0, + "REPLACE_STATEMENT": 7, + "BIND_PLACEHOLDER": 22 + }, + "shares": { + "STOP": 0.18783068783068782, + "REPLACE": 0.1111111111111111, + "ADD": 0.17724867724867724, + "REMOVE": 0.18253968253968253, + "ADD_CONTAINER": 0.013227513227513227, + "REMOVE_CONTAINER": 0.18253968253968253, + "INSERT_SUBTREE": 0.018518518518518517, + "REPLACE_SUBTREE": 0.05026455026455026, + "INSERT_STATEMENT": 0.0, + "REPLACE_STATEMENT": 0.018518518518518517, + "BIND_PLACEHOLDER": 0.0582010582010582 + }, + "inverse_action_weights": null + }, + "decode_demand": { + "overall": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 11092, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + }, + "by_source": { + "fixture_00": { + "visited": { + "BIND_PLACEHOLDER": 576, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 15, + "REPLACE": 3, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 297, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 9, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 185, + "invalid_result": 1, + "leaf_container_mismatch": 1, + "not_bindable": 252, + "not_canonical_subtree": 288, + "not_container": 15, + "not_leaf_component": 144, + "not_removable": 6, + "target_not_canonical": 2 + }, + "visited_total": 918, + "dead_total": 894, + "dead_candidate_rate": 0.9738562091503268, + "visited_shares": { + "BIND_PLACEHOLDER": 0.6274509803921569, + "INSERT_STATEMENT": 0.013071895424836602, + "REMOVE": 0.007625272331154684, + "REMOVE_CONTAINER": 0.016339869281045753, + "REPLACE": 0.0032679738562091504, + "REPLACE_STATEMENT": 0.002178649237472767, + "REPLACE_SUBTREE": 0.3235294117647059, + "STOP": 0.006535947712418301 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_01": { + "visited": { + "ADD": 2, + "BIND_PLACEHOLDER": 648, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 17, + "REMOVE": 3, + "REMOVE_CONTAINER": 14, + "REPLACE": 2, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 1022, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 3, + "REPLACE_SUBTREE": 230, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 17, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 7, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 407, + "invalid_result": 73, + "leaf_container_mismatch": 1, + "not_bindable": 432, + "not_canonical_subtree": 720, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 2, + "parent_or_comp_precondition": 17, + "target_not_canonical": 2 + }, + "visited_total": 1737, + "dead_total": 1701, + "dead_candidate_rate": 0.9792746113989638, + "visited_shares": { + "ADD": 0.0011514104778353484, + "BIND_PLACEHOLDER": 0.37305699481865284, + "INSERT_STATEMENT": 0.010362694300518135, + "INSERT_SUBTREE": 0.00978698906160046, + "REMOVE": 0.0017271157167530224, + "REMOVE_CONTAINER": 0.008059873344847437, + "REPLACE": 0.0011514104778353484, + "REPLACE_STATEMENT": 0.0011514104778353484, + "REPLACE_SUBTREE": 0.588370754173863, + "STOP": 0.0051813471502590676 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_02": { + "visited": { + "BIND_PLACEHOLDER": 504, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1431, + "dead_total": 1407, + "dead_candidate_rate": 0.9832285115303984, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3522012578616352, + "INSERT_STATEMENT": 0.008385744234800839, + "REMOVE": 0.004891684136967156, + "REMOVE_CONTAINER": 0.007686932215234102, + "REPLACE": 0.011879804332634521, + "REPLACE_STATEMENT": 0.0041928721174004195, + "REPLACE_SUBTREE": 0.6065688329839273, + "STOP": 0.0041928721174004195 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_03": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 8, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 72, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 75, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 8 + }, + "visited_total": 1397, + "dead_total": 1373, + "dead_candidate_rate": 0.9828203292770222, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3350035790980673, + "INSERT_STATEMENT": 0.008589835361488905, + "REMOVE": 0.005010737294201861, + "REMOVE_CONTAINER": 0.007874015748031496, + "REPLACE": 0.012168933428775949, + "REPLACE_STATEMENT": 0.00572655690765927, + "REPLACE_SUBTREE": 0.6213314244810307, + "STOP": 0.004294917680744453 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_04": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1395, + "dead_total": 1371, + "dead_candidate_rate": 0.9827956989247312, + "visited_shares": { + "BIND_PLACEHOLDER": 0.33548387096774196, + "INSERT_STATEMENT": 0.008602150537634409, + "REMOVE": 0.005017921146953405, + "REMOVE_CONTAINER": 0.007885304659498209, + "REPLACE": 0.012186379928315413, + "REPLACE_STATEMENT": 0.004301075268817204, + "REPLACE_SUBTREE": 0.6222222222222222, + "STOP": 0.004301075268817204 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_05": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 48, + "REMOVE": 4, + "REMOVE_CONTAINER": 9, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 9, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 50, + "target_not_canonical": 4 + }, + "visited_total": 1226, + "dead_total": 1194, + "dead_candidate_rate": 0.9738988580750407, + "visited_shares": { + "ADD": 0.0032626427406199023, + "BIND_PLACEHOLDER": 0.44045676998368677, + "INSERT_STATEMENT": 0.013050570962479609, + "INSERT_SUBTREE": 0.03915171288743882, + "REMOVE": 0.0032626427406199023, + "REMOVE_CONTAINER": 0.00734094616639478, + "REPLACE": 0.004893964110929853, + "REPLACE_STATEMENT": 0.0032626427406199023, + "REPLACE_SUBTREE": 0.47879282218597063, + "STOP": 0.0065252854812398045 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + }, + "fixture_06": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 720, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 33, + "REMOVE": 4, + "REMOVE_CONTAINER": 14, + "REPLACE": 5, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 1019, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 332, + "invalid_result": 38, + "leaf_container_mismatch": 2, + "not_bindable": 396, + "not_canonical_subtree": 864, + "not_container": 12, + "not_leaf_component": 108, + "not_removable": 3, + "parent_or_comp_precondition": 32, + "target_not_canonical": 4 + }, + "visited_total": 1827, + "dead_total": 1791, + "dead_candidate_rate": 0.9802955665024631, + "visited_shares": { + "ADD": 0.0005473453749315818, + "BIND_PLACEHOLDER": 0.39408866995073893, + "INSERT_STATEMENT": 0.009852216748768473, + "INSERT_SUBTREE": 0.0180623973727422, + "REMOVE": 0.0021893814997263274, + "REMOVE_CONTAINER": 0.007662835249042145, + "REPLACE": 0.002736726874657909, + "REPLACE_STATEMENT": 0.0021893814997263274, + "REPLACE_SUBTREE": 0.5577449370552818, + "STOP": 0.0049261083743842365 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_07": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 40, + "REMOVE": 4, + "REMOVE_CONTAINER": 10, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 10, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 42, + "target_not_canonical": 4 + }, + "visited_total": 1219, + "dead_total": 1187, + "dead_candidate_rate": 0.9737489745693191, + "visited_shares": { + "ADD": 0.003281378178835111, + "BIND_PLACEHOLDER": 0.44298605414273995, + "INSERT_STATEMENT": 0.013125512715340444, + "INSERT_SUBTREE": 0.03281378178835111, + "REMOVE": 0.003281378178835111, + "REMOVE_CONTAINER": 0.008203445447087777, + "REPLACE": 0.004922067268252666, + "REPLACE_STATEMENT": 0.003281378178835111, + "REPLACE_SUBTREE": 0.4815422477440525, + "STOP": 0.006562756357670222 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + } + }, + "by_suite": { + "train": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 0, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + } + } + }, + "calibration": { + "per_action": { + "STOP": { + "target_share": 0.18783068783068782, + "visited_share": 0.005201793721973094, + "gap": -0.18262889410871472 + }, + "REPLACE": { + "target_share": 0.1111111111111111, + "visited_share": 0.006547085201793722, + "gap": -0.10456402590931738 + }, + "ADD": { + "target_share": 0.17724867724867724, + "visited_share": 0.0009865470852017937, + "gap": -0.17626213016347544 + }, + "REMOVE": { + "target_share": 0.18253968253968253, + "visited_share": 0.0038565022421524664, + "gap": -0.17868318029753005 + }, + "ADD_CONTAINER": { + "target_share": 0.013227513227513227, + "visited_share": 0.0, + "gap": -0.013227513227513227 + }, + "REMOVE_CONTAINER": { + "target_share": 0.18253968253968253, + "visited_share": 0.008520179372197309, + "gap": -0.1740195031674852 + }, + "INSERT_SUBTREE": { + "target_share": 0.018518518518518517, + "visited_share": 0.012376681614349776, + "gap": -0.006141836904168741 + }, + "REPLACE_SUBTREE": { + "target_share": 0.05026455026455026, + "visited_share": 0.5485201793721973, + "gap": 0.4982556291076471 + }, + "INSERT_STATEMENT": { + "target_share": 0.0, + "visited_share": 0.010403587443946188, + "gap": 0.010403587443946188 + }, + "REPLACE_STATEMENT": { + "target_share": 0.018518518518518517, + "visited_share": 0.003228699551569507, + "gap": -0.01528981896694901 + }, + "BIND_PLACEHOLDER": { + "target_share": 0.0582010582010582, + "visited_share": 0.4003587443946188, + "gap": 0.34215768619356063 + } + }, + "mean_abs_deviation": 0.1546939823173007 + }, + "loss_reweighting_prediction": { + "predicted": true, + "rule": "predict reweighting iff applicable-ADD recall < 0.5 or |ADD visited share - ADD target share| > 0.2", + "add_share_gap": -0.17626213016347544, + "applicable_add_recall": 0.10344827586206896 + } + }, + "loss_decision": { + "audit_prediction": { + "predicted": true, + "rule": "predict reweighting iff applicable-ADD recall < 0.5 or |ADD visited share - ADD target share| > 0.2", + "add_share_gap": -0.17626213016347544, + "applicable_add_recall": 0.10344827586206896 + }, + "decision": "not_added_preregistered_cells_unchanged", + "rationale": "The audit's preregistered rule predicted reweighting=True. The matched 2x2 was preregistered without a loss arm; adding one post-hoc would break lever isolation, so it is deferred to a follow-up preregistered cell rather than combined here." + }, + "honesty": "Fixture-scale matched cells; decode-demand metrics come from SLM-310 per-proposal telemetry (visited = enumerated candidates the decode loop actually considered). A fixture verdict is wiring/distribution evidence, not a production ship claim. Negative results are retained per lever, never combined.", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "ae5448c594233b304ea22720604a962ad041fe63", + "code_dirty": true, + "components": { + "harness.experiments.slm310_action_alignment": "v1", + "harness.experiments.slm299_edit_reachability": "v4" + }, + "stamped_at": "2026-07-24T22:07:47.206210+00:00" + } +} diff --git a/docs/design/iter-slm310-action-alignment-20260724.md b/docs/design/iter-slm310-action-alignment-20260724.md new file mode 100644 index 000000000..7ab2a51f6 --- /dev/null +++ b/docs/design/iter-slm310-action-alignment-20260724.md @@ -0,0 +1,38 @@ +# SLM-310 (LAR2-03): action alignment — corruption sampler × STOP-slot accounting + +**Verdicts: corruption=ADD-balanced `adopted`, STOP-slot corrected `adopted`** (fixture-scale matched cells; not a ship claim) + +## Preregistered thresholds (locked before results) + +- T1: ADD training-target share gain >= 0.1 (both STOP arms) +- T2: ADD-balanced never reduces valid-final rate +- T3: corrected STOP accounting consumes <= legacy STOP budget and never reduces valid-final rate +- T4: lever isolation is structural (single-lever config diffs) +- rule: per lever: adopted iff its thresholds hold in both cells of the matched pair, else rejected; levers are never combined + +## Cells (levers isolated) + +| cell | corruption | STOP accounting | dead-candidate rate | applicable-ADD recall | ADD target share | verifier calls | STOP budget | valid-final | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| cell_a | default | legacy | 0.9792 | 0.1034 | 0.1772 | 11092 | 58 | 1.0000 | +| cell_b | ADD-balanced | legacy | 0.9799 | 0.1622 | 0.3275 | 14629 | 74 | 1.0000 | +| cell_c | default | corrected | 0.9792 | 0.1034 | 0.1772 | 11092 | 58 | 1.0000 | +| cell_d | ADD-balanced | corrected | 0.9799 | 0.1622 | 0.3275 | 14630 | 73 | 1.0000 | + +## Threshold checks + +- T1 gains: {'cell_b_minus_cell_a': 0.15020724214678874, 'cell_d_minus_cell_c': 0.15020724214678874} → ok=True +- T2 valid-final: {'cell_b_vs_cell_a': True, 'cell_d_vs_cell_c': True} → ok=True +- T3 STOP accounting: {'cell_c_vs_cell_a': {'stop_budget_legacy': 58, 'stop_budget_corrected': 58, 'budget_ok': True, 'valid_final_ok': True}, 'cell_d_vs_cell_b': {'stop_budget_legacy': 74, 'stop_budget_corrected': 73, 'budget_ok': True, 'valid_final_ok': True}} → ok=True +- T4 isolation: {'b_minus_a': ['corruption'], 'c_minus_a': ['stop'], 'd_minus_b': ['stop'], 'd_minus_c': ['corruption']} → ok=True + +## Baseline distribution audit (cell A) + +- dead-candidate rate 0.9792, applicable-ADD recall 0.1034, calibration MAD 0.1547 +- top rejection reasons: {'duplicate_state': 1883, 'invalid_result': 206, 'leaf_container_mismatch': 25, 'no_op': 6, 'not_bindable': 2628, 'not_canonical_subtree': 5328, 'not_container': 90, 'not_leaf_component': 540, 'not_removable': 35, 'parent_or_comp_precondition': 141, 'target_not_canonical': 36} +- loss-reweighting prediction: True (predict reweighting iff applicable-ADD recall < 0.5 or |ADD visited share - ADD target share| > 0.2) +- decision: not_added_preregistered_cells_unchanged — The audit's preregistered rule predicted reweighting=True. The matched 2x2 was preregistered without a loss arm; adding one post-hoc would break lever isolation, so it is deferred to a follow-up preregistered cell rather than combined here. + +## Honesty + +Fixture-scale matched cells; decode-demand metrics come from SLM-310 per-proposal telemetry (visited = enumerated candidates the decode loop actually considered). A fixture verdict is wiring/distribution evidence, not a production ship claim. Negative results are retained per lever, never combined. diff --git a/scripts/run_slm310_action_alignment.py b/scripts/run_slm310_action_alignment.py new file mode 100644 index 000000000..bb5257f4d --- /dev/null +++ b/scripts/run_slm310_action_alignment.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +"""SLM-310 (LAR2-03): matched fixture experiment — action alignment in X22. + +Question: does aligning the corruption sampler with a declared inverse-action +distribution (ADD-balanced) close the training-target vs decode-demand gap, +and does corrected STOP-slot accounting (STOP consumes an expand_per_state +slot only when its frozen candidate is retained) change search efficiency — +each lever isolated, at equal model / data / optimizer budget? + +Cells (2×2, levers isolated; A is the baseline): + +- A: default corruption sampler × legacy STOP accounting +- B: ADD-balanced corruption sampler × legacy STOP accounting +- C: default corruption sampler × corrected STOP accounting +- D: ADD-balanced corruption sampler × corrected STOP accounting + +Preregistered (written into the output payload BEFORE any result; locked — +do not edit after outcomes are visible): + +- T1 (corruption lever): within each STOP arm, the ADD-balanced cell must + raise the ADD training-target share by >= 0.10 absolute vs the default + cell (sampler hits its declared distribution); +- T2 (corruption lever): within each STOP arm, the ADD-balanced cell must + not reduce the valid-final rate below the default cell's (no semantic + regression); +- T3 (STOP lever): within each corruption arm, the corrected cell must + consume strictly fewer or equal STOP budget slots than the legacy cell + (duplicate STOPs no longer burn budget) and must not reduce the + valid-final rate; +- T4 (isolation): B−A differs only in the sampler, C−A only in STOP + accounting — verified structurally by config diff, not by narration. +- Verdict per lever: ``adopted`` iff its thresholds hold in BOTH cells of + the pair, else ``rejected``. A negative result is retained, never + combined. + +The distribution audit (training-target vs decode-demand) runs on the +baseline cell and its preregistered prediction decides whether a +class-balanced/focal action loss is warranted; the prediction and the +decision are recorded in the payload (no loss arm was added unless the audit +predicted it — see ``loss_decision``). + +Writes ``docs/design/iter-slm310-action-alignment-20260724.{json,md}``. + +Example: + python -m scripts.run_slm310_action_alignment --steps 8 +""" + +from __future__ import annotations + +import argparse +import json +import random +from pathlib import Path + +import torch + +from slm_training.dsl.parser import validate +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm310_action_alignment import ( + action_calibration, + decode_demand_distribution, + run_distribution_audit, + training_target_distribution, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_STOP, + TreeEditDiffusionConfig, + TreeEditDiffusionModel, +) +from slm_training.versioning import build_version_stamp + +EXPERIMENT_ID = "slm310-action-alignment" +DEFAULT_JSON_OUT = Path("docs/design/iter-slm310-action-alignment-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm310-action-alignment-20260724.md") + +# PREREGISTERED thresholds (locked before any run; deviations are append-only +# and exploratory). +PREREGISTERED_THRESHOLDS = { + "t1_add_target_share_gain_min": 0.10, + "t2_valid_final_no_regression": True, + "t3_stop_budget_not_worse_and_valid_final_no_regression": True, + "t4_lever_isolation_structural": True, + "verdict_rule": ( + "per lever: adopted iff its thresholds hold in both cells of the " + "matched pair, else rejected; levers are never combined" + ), +} + +# Declared inverse-action distribution for the ADD-balanced corruption arm: +# ADD lifted well above its default ~10% share (only REMOVE mutations yield +# ADD inverses), all other reachable inverse actions uniform. +ADD_BALANCED_DISTRIBUTION = { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0, +} + +CELLS = { + "cell_a": {"corruption": None, "stop": "legacy"}, + "cell_b": {"corruption": ADD_BALANCED_DISTRIBUTION, "stop": "legacy"}, + "cell_c": {"corruption": None, "stop": "corrected"}, + "cell_d": {"corruption": ADD_BALANCED_DISTRIBUTION, "stop": "corrected"}, +} + +# Tiny fixture corpus (same shape as the SLM-308 matched experiment). +FIXTURE_PROGRAMS: list[tuple[str, str, list[str]]] = [ + ("hero with cta", 'root = Stack([t, b], "column")\nt = TextContent(":hero.title")\nb = Button(":cta.label")', [":hero.title", ":cta.label"]), + ("simple text", 'root = Stack([t], "column")\nt = TextContent(":body")', [":body"]), + ("form with input", 'root = Form([i, s], "column")\ni = TextInput(":form.email")\ns = Button(":form.submit")', [":form.email", ":form.submit"]), + ("card with image", 'root = Stack([c], "column")\nc = Card([im, t], "column")\nim = Image(":img.src")\nt = TextContent(":img.caption")', [":img.src", ":img.caption"]), + ("two texts", 'root = Stack([a, b], "column")\na = TextContent(":title")\nb = TextContent(":subtitle")', [":title", ":subtitle"]), + ("button only", 'root = Stack([b], "column")\nb = Button(":cta")', [":cta"]), + ("nested card", 'root = Stack([c], "column")\nc = Card([t], "column")\nt = TextContent(":card.body")', [":card.body"]), + ("form pair", 'root = Form([i], "column")\ni = TextInput(":q")', [":q"]), +] + + +def build_fixture_records() -> list[ExampleRecord]: + return [ + ExampleRecord( + id=f"fixture_{i:02d}", + prompt=prompt, + openui=openui, + placeholders=list(placeholders), + split="train", + ) + for i, (prompt, openui, placeholders) in enumerate(FIXTURE_PROGRAMS) + ] + + +def train_cell( + records: list[ExampleRecord], + corruption: dict[str, float] | None, + stop: str, + *, + steps: int, + batch_size: int, + seed: int, +) -> TreeEditDiffusionModel: + """Train one cell; cells share corpus, init seed, batch order, optimizer + budget, and value_label_mode — only the cell's lever(s) differ.""" + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + d_model=32, + n_heads=4, + context_layers=1, + denoiser_layers=1, + seed=seed, + max_chain=3, + value_label_mode="mutation_count", + context_backend="scratch", + corruption_action_distribution=corruption, + stop_slot_accounting=stop, + ) + model = TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=3e-3) + order = random.Random(0) # identical batch order across cells + model.train() + for _ in range(steps): + batch = [records[order.randrange(len(records))] for _ in range(batch_size)] + loss = model.training_loss(batch) + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +def score_cell( + model: TreeEditDiffusionModel, + records: list[ExampleRecord], + *, + audit_samples: int, + seed: int, +) -> dict: + """Per-cell metrics: dead-candidate rate, applicable-ADD recall, action + calibration, verifier calls, semantic outcomes (valid-final rate).""" + weights = model._inverse_action_weights() + training = training_target_distribution( + records, + model.space, + seed=seed, + samples=audit_samples, + max_chain=model.config.max_chain, + inverse_action_weights=weights, + ) + demand = decode_demand_distribution(model, records, include_outputs=True) + outputs = demand.pop("outputs") + valid_final = 0 + for text in outputs: + try: + validate(text) + valid_final += 1 + except Exception: # noqa: BLE001 + pass + calib = action_calibration(training, demand["overall"]) + overall = demand["overall"] + proposals = [ + p for ev in demand["evidence"] for p in ev.get("proposals", []) + ] + stop_budget = sum( + 1 for p in proposals if p["action"] == ACTION_STOP and p["consumed_budget"] + ) + stop_visited = sum(1 for p in proposals if p["action"] == ACTION_STOP) + return { + "training_targets": training, + "decode_demand": { + "overall": overall, + "by_suite": demand["by_suite"], + "by_source": demand["by_source"], + }, + "calibration": calib, + "dead_candidate_rate": overall["dead_candidate_rate"], + "applicable_add_recall": overall["applicable_add_recall"], + "verifier_calls": overall["verifier_calls"], + "stop_budget_consumed": stop_budget, + "stop_proposals_visited": stop_visited, + "valid_final_rate": valid_final / max(len(records), 1), + "n_records": len(records), + "training_metrics": model.last_training_metrics, + } + + +def build_report( + records: list[ExampleRecord], + *, + steps: int, + batch_size: int, + seed: int, + audit_samples: int, +) -> dict: + cells: dict[str, dict] = {} + for cell, spec in CELLS.items(): + model = train_cell( + records, + spec["corruption"], + spec["stop"], + steps=steps, + batch_size=batch_size, + seed=seed, + ) + metrics = score_cell(model, records, audit_samples=audit_samples, seed=seed) + cells[cell] = {"levers": spec, "metrics": metrics} + + # T4: lever isolation is structural — pairwise config diffs must name + # exactly one lever. + def lever_diff(x: str, y: str) -> list[str]: + return [ + key + for key in ("corruption", "stop") + if CELLS[x][key] != CELLS[y][key] + ] + + isolation = { + "b_minus_a": lever_diff("cell_b", "cell_a"), + "c_minus_a": lever_diff("cell_c", "cell_a"), + "d_minus_b": lever_diff("cell_d", "cell_b"), + "d_minus_c": lever_diff("cell_d", "cell_c"), + } + t4_ok = ( + isolation["b_minus_a"] == ["corruption"] + and isolation["c_minus_a"] == ["stop"] + and isolation["d_minus_b"] == ["stop"] + and isolation["d_minus_c"] == ["corruption"] + ) + + thr = PREREGISTERED_THRESHOLDS + checks: dict[str, dict] = {} + # T1/T2: corruption lever, compared within each STOP arm (A→B, C→D). + t1 = {} + t2 = {} + for base, balanced in (("cell_a", "cell_b"), ("cell_c", "cell_d")): + a_add = cells[base]["metrics"]["training_targets"]["shares"]["ADD"] + b_add = cells[balanced]["metrics"]["training_targets"]["shares"]["ADD"] + t1[f"{balanced}_minus_{base}"] = b_add - a_add + a_vf = cells[base]["metrics"]["valid_final_rate"] + b_vf = cells[balanced]["metrics"]["valid_final_rate"] + t2[f"{balanced}_vs_{base}"] = b_vf >= a_vf + t1_ok = all(g >= thr["t1_add_target_share_gain_min"] for g in t1.values()) + t2_ok = all(t2.values()) + # T3: STOP lever, compared within each corruption arm (A→C, B→D). + t3 = {} + for legacy, corrected in (("cell_a", "cell_c"), ("cell_b", "cell_d")): + leg = cells[legacy]["metrics"] + cor = cells[corrected]["metrics"] + t3[f"{corrected}_vs_{legacy}"] = { + "stop_budget_legacy": leg["stop_budget_consumed"], + "stop_budget_corrected": cor["stop_budget_consumed"], + "budget_ok": cor["stop_budget_consumed"] <= leg["stop_budget_consumed"], + "valid_final_ok": cor["valid_final_rate"] >= leg["valid_final_rate"], + } + t3_ok = all(v["budget_ok"] and v["valid_final_ok"] for v in t3.values()) + + checks = { + "t1_add_target_share_gains": t1, + "t1_ok": t1_ok, + "t2_valid_final": t2, + "t2_ok": t2_ok, + "t3_stop_accounting": t3, + "t3_ok": t3_ok, + "t4_lever_isolation": isolation, + "t4_ok": t4_ok, + } + verdicts = { + "corruption_add_balanced": "adopted" if (t1_ok and t2_ok) else "rejected", + "stop_slot_corrected": "adopted" if t3_ok else "rejected", + } + + # Distribution audit on the baseline cell decides whether a + # class-balanced/focal action loss is warranted (preregistered rule). + audit_model = train_cell( + records, None, "legacy", steps=steps, batch_size=batch_size, seed=seed + ) + audit = run_distribution_audit( + audit_model, records, seed=seed, samples=audit_samples, max_chain=3 + ) + predicted = audit["loss_reweighting_prediction"]["predicted"] + loss_decision = { + "audit_prediction": audit["loss_reweighting_prediction"], + "decision": ( + "not_added" + if not predicted + else "not_added_preregistered_cells_unchanged" + ), + "rationale": ( + "The audit's preregistered rule predicted " + f"reweighting={predicted}. The matched 2x2 was preregistered " + "without a loss arm; adding one post-hoc would break lever " + "isolation, so it is deferred to a follow-up preregistered cell " + "rather than combined here." + ), + } + + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-310", + "question": ( + "Does ADD-balanced corruption sampling close the train/demand " + "action gap, and does corrected STOP-slot accounting improve " + "search budget use — each lever isolated at fixture scale?" + ), + "preregistered_thresholds": PREREGISTERED_THRESHOLDS, + "config": { + "steps": steps, + "batch_size": batch_size, + "seed": seed, + "audit_samples": audit_samples, + "n_records": len(records), + "value_label_mode": "mutation_count", + "add_balanced_distribution": ADD_BALANCED_DISTRIBUTION, + "cells": CELLS, + }, + "cells": cells, + "threshold_checks": checks, + "verdicts": verdicts, + "baseline_audit": audit, + "loss_decision": loss_decision, + "honesty": ( + "Fixture-scale matched cells; decode-demand metrics come from " + "SLM-310 per-proposal telemetry (visited = enumerated candidates " + "the decode loop actually considered). A fixture verdict is " + "wiring/distribution evidence, not a production ship claim. " + "Negative results are retained per lever, never combined." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm310_action_alignment", + "harness.experiments.slm299_edit_reachability", + ) + return payload + + +def render_markdown(payload: dict) -> str: + cells = payload["cells"] + thr = payload["preregistered_thresholds"] + checks = payload["threshold_checks"] + + def fmt(x: object) -> str: + return "n/a" if x is None else (f"{x:.4f}" if isinstance(x, float) else str(x)) + + lines = [ + "# SLM-310 (LAR2-03): action alignment — corruption sampler × STOP-slot accounting", + "", + f"**Verdicts: corruption=ADD-balanced `{payload['verdicts']['corruption_add_balanced']}`, " + f"STOP-slot corrected `{payload['verdicts']['stop_slot_corrected']}`** " + "(fixture-scale matched cells; not a ship claim)", + "", + "## Preregistered thresholds (locked before results)", + "", + f"- T1: ADD training-target share gain >= {thr['t1_add_target_share_gain_min']} (both STOP arms)", + "- T2: ADD-balanced never reduces valid-final rate", + "- T3: corrected STOP accounting consumes <= legacy STOP budget and never reduces valid-final rate", + "- T4: lever isolation is structural (single-lever config diffs)", + f"- rule: {thr['verdict_rule']}", + "", + "## Cells (levers isolated)", + "", + "| cell | corruption | STOP accounting | dead-candidate rate | applicable-ADD recall | ADD target share | verifier calls | STOP budget | valid-final |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for cell, entry in cells.items(): + m = entry["metrics"] + lev = entry["levers"] + lines.append( + f"| {cell} | {'ADD-balanced' if lev['corruption'] else 'default'} " + f"| {lev['stop']} | {fmt(m['dead_candidate_rate'])} " + f"| {fmt(m['applicable_add_recall'])} " + f"| {fmt(m['training_targets']['shares']['ADD'])} " + f"| {m['verifier_calls']} | {m['stop_budget_consumed']} " + f"| {fmt(m['valid_final_rate'])} |" + ) + lines += [ + "", + "## Threshold checks", + "", + f"- T1 gains: {checks['t1_add_target_share_gains']} → ok={checks['t1_ok']}", + f"- T2 valid-final: {checks['t2_valid_final']} → ok={checks['t2_ok']}", + f"- T3 STOP accounting: {checks['t3_stop_accounting']} → ok={checks['t3_ok']}", + f"- T4 isolation: {checks['t4_lever_isolation']} → ok={checks['t4_ok']}", + "", + "## Baseline distribution audit (cell A)", + "", + ] + audit = payload["baseline_audit"] + lines.append( + f"- dead-candidate rate {fmt(audit['decode_demand']['overall']['dead_candidate_rate'])}, " + f"applicable-ADD recall {fmt(audit['decode_demand']['overall']['applicable_add_recall'])}, " + f"calibration MAD {fmt(audit['calibration']['mean_abs_deviation'])}" + ) + pred = payload["loss_decision"]["audit_prediction"] + lines += [ + f"- top rejection reasons: {audit['decode_demand']['overall']['rejection_reasons']}", + f"- loss-reweighting prediction: {pred['predicted']} ({pred['rule']})", + f"- decision: {payload['loss_decision']['decision']} — {payload['loss_decision']['rationale']}", + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=8) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--audit-samples", type=int, default=400) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + args = parser.parse_args(argv) + + records = build_fixture_records() + payload = build_report( + records, + steps=args.steps, + batch_size=args.batch_size, + seed=args.seed, + audit_samples=args.audit_samples, + ) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"verdicts={payload['verdicts']} " + f"t1={payload['threshold_checks']['t1_ok']} " + f"t3={payload['threshold_checks']['t3_ok']} " + f"-> {args.json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm310_action_alignment.py b/src/slm_training/harnesses/experiments/slm310_action_alignment.py new file mode 100644 index 000000000..8e9bddbca --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm310_action_alignment.py @@ -0,0 +1,330 @@ +"""SLM-310 (LAR2-03): action-alignment audit for the X22 tree-edit model. + +Question: does the inverse-edit supervision distribution (what training +targets ask the policy to repair) match the decode-demand distribution (what +the value-guided beam actually proposes, finds applicable, and selects)? + +Two measurement sides, both fixture-scale and deterministic: + +- ``training_target_distribution`` replays the training-time sampling loop + (a ``stop_fraction`` share of clean-state STOP targets + 1..max_chain + forward-noise chains whose last inverse edit is the supervised target) and + counts inverse actions. +- ``decode_demand_distribution`` decodes every record and aggregates the + SLM-310 per-proposal telemetry (``evidence["proposals"]`` / + ``evidence["states"]``): per action, how many enumerated candidates were + visited, applicable, and selected — overall, by source record, by suite. + +Derived metrics: + +- dead-candidate rate: visited proposals with a rejection reason + (inapplicable or duplicate) over all visited proposals; +- applicable-ADD recall: over the exact decode states where at least one + ADD edit is applicable in the full edit space, the fraction where the + decode loop visited at least one applicable ADD proposal; +- action calibration: per-action decode-visited share vs training-target + share (mean absolute deviation over actions). +""" + +from __future__ import annotations + +import random +from collections import Counter +from typing import Any + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.model_build.plugin import GenerationRequest +from slm_training.models.tree_edit_diffusion import ( + ACTION_ADD, + ACTION_NAMES, + ACTION_STOP, + Edit, + LEAF_COMPONENTS, + MAX_SLOTS, + TreeEditDiffusionModel, + TreeEditSpace, + parse_statements, +) + +#: Preregistered audit rule (locked before any experiment run): recommend a +#: class-balanced/focal action loss only when the measured misalignment is +#: large enough — applicable-ADD recall below this floor OR an ADD +#: train/demand share gap above this bound. +ADD_RECALL_FLOOR = 0.5 +ADD_SHARE_GAP = 0.20 + + +def _inventory(record: ExampleRecord) -> list[str]: + return [ + p if p.startswith(":") else f":{p}" + for p in (record.placeholders or []) + ][:MAX_SLOTS] + + +def training_target_distribution( + records: list[ExampleRecord], + space: TreeEditSpace, + *, + seed: int = 0, + samples: int = 400, + max_chain: int = 4, + inverse_action_weights: dict[int, float] | None = None, + stop_fraction: float = 0.2, +) -> dict[str, Any]: + """Replay the training-time target sampler and count inverse actions. + + Mirrors ``TreeEditDiffusionModel.training_loss``: with probability + ``stop_fraction`` the target is STOP (clean state); otherwise a + 1..max_chain forward-noise chain is applied and the last inverse edit is + the target. Records whose program fails to parse are skipped (as in + training). Deterministic under ``seed``. + """ + rng = random.Random(seed) + parsable = [r for r in records if parse_statements(r.openui or "")] + counts: Counter[int] = Counter() + attempted = 0 + for _ in range(samples): + if not parsable: + break + record = parsable[rng.randrange(len(parsable))] + statements = parse_statements(record.openui) + assert statements is not None + inventory = _inventory(record) + attempted += 1 + if rng.random() < stop_fraction: + counts[ACTION_STOP] += 1 + continue + k = rng.randint(1, max_chain) + current = statements + inverse: Edit | None = None + for _ in range(k): + step = space.sample_mutation( + current, + inventory, + rng, + inverse_action_weights=inverse_action_weights, + ) + if step is None: + break + current, inverse = step + if inverse is not None: + counts[inverse.action] += 1 + total = sum(counts.values()) + return { + "samples": attempted, + "n_targets": total, + "counts": { + ACTION_NAMES[a]: counts.get(a, 0) for a in range(len(ACTION_NAMES)) + }, + "shares": { + ACTION_NAMES[a]: (counts.get(a, 0) / total if total else 0.0) + for a in range(len(ACTION_NAMES)) + }, + "inverse_action_weights": ( + None + if inverse_action_weights is None + else { + ACTION_NAMES[a]: w for a, w in sorted(inverse_action_weights.items()) + } + ), + } + + +def applicable_add_exists( + statements: list, inventory: list[str], space: TreeEditSpace +) -> bool: + """True when at least one ADD edit is applicable to the state.""" + leaf_comps = [ + i for i, c in enumerate(space.components) if c in LEAF_COMPONENTS + ] + for stmt in range(len(statements)): + for comp in leaf_comps: + for slot in range(min(len(inventory), MAX_SLOTS)): + if space.apply( + statements, Edit(ACTION_ADD, stmt, comp, slot), inventory + ): + return True + return False + + +def _new_bucket() -> dict[str, Any]: + return { + "visited": Counter(), + "applicable": Counter(), + "selected": Counter(), + "dead": Counter(), + "rejection_reasons": Counter(), + "verifier_calls": 0, + "states_with_applicable_add": 0, + "states_recalled": 0, + } + + +def _add_proposals(bucket: dict[str, Any], proposals: list[dict[str, Any]]) -> None: + for p in proposals: + name = p["action_name"] + bucket["visited"][name] += 1 + if p["applicable"]: + bucket["applicable"][name] += 1 + if p["selected"]: + bucket["selected"][name] += 1 + if p["rejection_reason"] is not None: + bucket["dead"][name] += 1 + bucket["rejection_reasons"][p["rejection_reason"]] += 1 + + +def _add_recall( + bucket: dict[str, Any], + ev: dict[str, Any], + inventory: list[str], + space: TreeEditSpace, +) -> None: + """Exact applicable-ADD recall over the states this decode expanded.""" + proposals = ev.get("proposals", []) + visited_add_states = { + (p["step"], p["beam_row"]) + for p in proposals + if p["action"] == ACTION_ADD and p["applicable"] + } + for entry in ev.get("states", []): + statements = parse_statements(entry["source"]) + if statements is None: + continue + if not applicable_add_exists(statements, inventory, space): + continue + bucket["states_with_applicable_add"] += 1 + if (entry["step"], entry["beam_row"]) in visited_add_states: + bucket["states_recalled"] += 1 + + +def _finalize(bucket: dict[str, Any]) -> dict[str, Any]: + visited = sum(bucket["visited"].values()) + dead = sum(bucket["dead"].values()) + n_add_states = bucket["states_with_applicable_add"] + return { + "visited": dict(sorted(bucket["visited"].items())), + "applicable": dict(sorted(bucket["applicable"].items())), + "selected": dict(sorted(bucket["selected"].items())), + "rejection_reasons": dict(sorted(bucket["rejection_reasons"].items())), + "visited_total": visited, + "dead_total": dead, + "dead_candidate_rate": (dead / visited) if visited else None, + "visited_shares": { + name: (count / visited if visited else 0.0) + for name, count in sorted(bucket["visited"].items()) + }, + "verifier_calls": bucket["verifier_calls"], + "states_with_applicable_add": n_add_states, + "states_recalled": bucket["states_recalled"], + "applicable_add_recall": ( + bucket["states_recalled"] / n_add_states if n_add_states else None + ), + } + + +def decode_demand_distribution( + model: TreeEditDiffusionModel, + records: list[ExampleRecord], + *, + include_outputs: bool = False, +) -> dict[str, Any]: + """Decode every record and aggregate proposal telemetry by action — + overall, by source record, and by suite (split).""" + requests = [ + GenerationRequest( + prompt=r.prompt, + slot_contract=tuple(_inventory(r)), + design_md=r.design_md, + ) + for r in records + ] + outputs = model.generate_batch_requests(requests) + evidence = model.consume_generation_evidence() + overall = _new_bucket() + by_source: dict[str, Any] = {} + by_suite: dict[str, Any] = {} + for record, ev in zip(records, evidence): + inventory = _inventory(record) + proposals = ev.get("proposals", []) + overall["verifier_calls"] += ev.get("verifier_calls", 0) + _add_proposals(overall, proposals) + _add_recall(overall, ev, inventory, model.space) + src = by_source.setdefault(record.id, _new_bucket()) + _add_proposals(src, proposals) + _add_recall(src, ev, inventory, model.space) + suite = by_suite.setdefault(record.split or "unspecified", _new_bucket()) + _add_proposals(suite, proposals) + _add_recall(suite, ev, inventory, model.space) + result = { + "overall": _finalize(overall), + "by_source": {k: _finalize(v) for k, v in sorted(by_source.items())}, + "by_suite": {k: _finalize(v) for k, v in sorted(by_suite.items())}, + } + if include_outputs: + result["outputs"] = outputs + result["evidence"] = evidence + return result + + +def action_calibration( + training: dict[str, Any], demand_overall: dict[str, Any] +) -> dict[str, Any]: + """Per-action training-target share vs decode visited share + MAD.""" + target = training["shares"] + visited = demand_overall["visited_shares"] + rows = {} + mad_terms = [] + for name in ACTION_NAMES: + t = target.get(name, 0.0) + d = visited.get(name, 0.0) + rows[name] = {"target_share": t, "visited_share": d, "gap": d - t} + mad_terms.append(abs(d - t)) + return { + "per_action": rows, + "mean_abs_deviation": sum(mad_terms) / len(mad_terms), + } + + +def run_distribution_audit( + model: TreeEditDiffusionModel, + records: list[ExampleRecord], + *, + seed: int = 0, + samples: int = 400, + max_chain: int = 4, + inverse_action_weights: dict[int, float] | None = None, +) -> dict[str, Any]: + """Full audit: training-target vs decode-demand distributions, dead + candidates, applicable-ADD recall, calibration, and the preregistered + loss-reweighting prediction.""" + training = training_target_distribution( + records, + model.space, + seed=seed, + samples=samples, + max_chain=max_chain, + inverse_action_weights=inverse_action_weights, + ) + demand = decode_demand_distribution(model, records) + calib = action_calibration(training, demand["overall"]) + recall = demand["overall"]["applicable_add_recall"] + add_gap = calib["per_action"]["ADD"]["gap"] + predicted = bool( + (recall is not None and recall < ADD_RECALL_FLOOR) + or abs(add_gap) > ADD_SHARE_GAP + ) + return { + "training_targets": training, + "decode_demand": demand, + "calibration": calib, + "loss_reweighting_prediction": { + "predicted": predicted, + "rule": ( + f"predict reweighting iff applicable-ADD recall < " + f"{ADD_RECALL_FLOOR} or |ADD visited share - ADD target " + f"share| > {ADD_SHARE_GAP}" + ), + "add_share_gap": add_gap, + "applicable_add_recall": recall, + }, + } diff --git a/src/slm_training/models/tree_edit_diffusion.py b/src/slm_training/models/tree_edit_diffusion.py index 9d973f5b1..6b34b1e36 100644 --- a/src/slm_training/models/tree_edit_diffusion.py +++ b/src/slm_training/models/tree_edit_diffusion.py @@ -63,6 +63,64 @@ ACTION_BIND_PLACEHOLDER = 10 # (re)bind a leaf's slot to an inventory placeholder N_ACTIONS = 11 +ACTION_NAMES: tuple[str, ...] = ( + "STOP", + "REPLACE", + "ADD", + "REMOVE", + "ADD_CONTAINER", + "REMOVE_CONTAINER", + "INSERT_SUBTREE", + "REPLACE_SUBTREE", + "INSERT_STATEMENT", + "REPLACE_STATEMENT", + "BIND_PLACEHOLDER", +) +ACTION_IDS: dict[str, int] = {name: i for i, name in enumerate(ACTION_NAMES)} + +# SLM-310: mutation kinds that can realize a given *inverse* (supervised +# repair) action. sample_mutation reweights its forward-noise choice by +# sampling the inverse action from a declared distribution and mapping back +# to a mutation kind. STOP is never an inverse action (unreachable). +INVERSE_TO_MUTATION_KINDS: dict[int, tuple[int, ...]] = { + ACTION_REPLACE: (ACTION_REPLACE,), + ACTION_ADD: (ACTION_REMOVE,), + ACTION_REMOVE: (ACTION_ADD, ACTION_INSERT_STATEMENT), + ACTION_ADD_CONTAINER: (ACTION_REMOVE_CONTAINER,), + ACTION_REMOVE_CONTAINER: (ACTION_ADD_CONTAINER, ACTION_INSERT_SUBTREE), + ACTION_INSERT_SUBTREE: (ACTION_REMOVE_CONTAINER,), + ACTION_REPLACE_SUBTREE: (ACTION_REPLACE_SUBTREE,), + ACTION_REPLACE_STATEMENT: (ACTION_REPLACE_STATEMENT,), + ACTION_BIND_PLACEHOLDER: (ACTION_BIND_PLACEHOLDER,), +} + +# SLM-310: reason codes emitted by TreeEditSpace.apply (out-param ``reason``) +# so decode-time proposal telemetry can report WHY a candidate was dead. +REASON_INDEX_OUT_OF_RANGE = "index_out_of_range" +REASON_NO_OP = "no_op" +REASON_LEAF_CONTAINER_MISMATCH = "leaf_container_mismatch" +REASON_MAX_STMTS = "max_stmts" +REASON_PARENT_OR_COMP = "parent_or_comp_precondition" +REASON_SLOT_OUT_OF_RANGE = "slot_out_of_range" +REASON_NOT_REMOVABLE = "not_removable" +REASON_UNREFERENCED_LEAF = "unreferenced_leaf" +REASON_TARGET_OUT_OF_RANGE = "target_out_of_range" +REASON_NOT_CONTAINER = "not_container" +REASON_SUBTREE_NOT_LEAF_ONLY = "subtree_not_leaf_only" +REASON_UNREFERENCED_CONTAINER = "unreferenced_container" +REASON_PAYLOAD_OUT_OF_RANGE = "payload_out_of_range" +REASON_PAYLOAD_NOT_LEAF = "payload_not_leaf" +REASON_NOT_CANONICAL_SUBTREE = "not_canonical_subtree" +REASON_STATEMENT_INVALID = "statement_invalid" +REASON_TARGET_NOT_CANONICAL = "target_not_canonical" +REASON_NOT_BINDABLE = "not_bindable" +REASON_NOT_LEAF_COMPONENT = "not_leaf_component" +REASON_UNKNOWN_ACTION = "unknown_action" +REASON_PRE_VALIDATE_REJECTED = "pre_validate_rejected" +REASON_INVALID_RESULT = "invalid_result" +# Decode-side (not from apply): candidate produced an already-seen state. +REASON_DUPLICATE_STATE = "duplicate_state" + MAX_STMTS = 24 MAX_SLOTS = 16 @@ -283,6 +341,7 @@ def apply( edit: Edit, inventory: list[str], pre_validate: Callable[[list[Statement]], bool] | None = None, + reason: list[str] | None = None, ) -> list[Statement] | None: """Apply one edit; None when inapplicable or invalid (fail closed). @@ -291,7 +350,16 @@ def apply( reachability analyzer to skip already-visited states). It can only reject, never accept: every accepted state is still re-validated through the real parser. + + ``reason`` (SLM-310) is an optional out-list: when the edit is + rejected, exactly one machine-readable rejection code is appended + (``REASON_*`` constants). Acceptance appends nothing. """ + def _fail(code: str) -> None: + if reason is not None: + reason.append(code) + return None + if edit.action == ACTION_STOP: return [Statement(**vars(s)) for s in statements] working = [ @@ -300,26 +368,26 @@ def apply( ] if edit.action == ACTION_REPLACE: if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] new_comp = self.components[edit.comp] leaf_like = not target.has_list if leaf_like != (new_comp in LEAF_COMPONENTS): - return None + return _fail(REASON_LEAF_CONTAINER_MISMATCH) if target.comp == new_comp: - return None + return _fail(REASON_NO_OP) target.comp = new_comp elif edit.action == ACTION_ADD: if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) if len(working) >= MAX_STMTS: - return None + return _fail(REASON_MAX_STMTS) parent = working[edit.stmt] comp = self.components[edit.comp] if not parent.has_list or comp not in LEAF_COMPONENTS: - return None + return _fail(REASON_PARENT_OR_COMP) if not inventory or not (0 <= edit.slot < len(inventory)): - return None + return _fail(REASON_SLOT_OUT_OF_RANGE) placeholder = inventory[edit.slot] if not placeholder.startswith(":"): placeholder = f":{placeholder}" @@ -336,10 +404,10 @@ def apply( ) elif edit.action == ACTION_REMOVE: if not (0 <= edit.stmt < len(working)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] if target.has_list or target.name == "root": - return None + return _fail(REASON_NOT_REMOVABLE) referenced = False for other in working: if target.name in other.children: @@ -349,22 +417,22 @@ def apply( # Unreferenced UI leaves stay immutable (old behavior); V0.5 # pack statements are unreferenced by construction and are # removable (inverse of INSERT_STATEMENT). - return None + return _fail(REASON_UNREFERENCED_LEAF) working = [s for s in working if s.name != target.name] elif edit.action == ACTION_ADD_CONTAINER: # Preconditions: parent is a container, MAX_STMTS bound, comp is a # container. The minted container starts EMPTY (leaf-only subtree) # so REMOVE_CONTAINER is an exact safe inverse. if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) if len(working) >= MAX_STMTS: - return None + return _fail(REASON_MAX_STMTS) parent = working[edit.stmt] comp = self.components[edit.comp] if not parent.has_list or comp not in CONTAINER_COMPONENTS: - return None + return _fail(REASON_PARENT_OR_COMP) if not (0 <= edit.target < len(CONTAINER_RESTS)): - return None + return _fail(REASON_TARGET_OUT_OF_RANGE) name = self.fresh_name(working) parent.children.append(name) working.append( @@ -380,18 +448,18 @@ def apply( # shapes the container-creating actions mint, so removal restores # the prior state exactly. Leaf children are dropped with it. if not (0 <= edit.stmt < len(working)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] if not target.has_list or target.name == "root": - return None + return _fail(REASON_NOT_CONTAINER) by_name = {s.name: s for s in working} if any( by_name.get(child) is None or by_name[child].has_list for child in target.children ): - return None + return _fail(REASON_SUBTREE_NOT_LEAF_ONLY) if not any(target.name in other.children for other in working): - return None + return _fail(REASON_UNREFERENCED_CONTAINER) drop = {target.name, *target.children} working = [ Statement( @@ -412,22 +480,22 @@ def apply( # comp is a container, payload indexes a leaf component, slot is # in inventory, MAX_STMTS bound. Inverse: REMOVE_CONTAINER. if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) if len(working) + 2 > MAX_STMTS: - return None + return _fail(REASON_MAX_STMTS) parent = working[edit.stmt] root_comp = self.components[edit.comp] if not parent.has_list or root_comp not in CONTAINER_COMPONENTS: - return None + return _fail(REASON_PARENT_OR_COMP) if not (0 <= edit.payload < len(self.components)): - return None + return _fail(REASON_PAYLOAD_OUT_OF_RANGE) leaf_comp = self.components[edit.payload] if leaf_comp not in LEAF_COMPONENTS: - return None + return _fail(REASON_PAYLOAD_NOT_LEAF) if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): - return None + return _fail(REASON_SLOT_OUT_OF_RANGE) if not (0 <= edit.target < len(CONTAINER_RESTS)): - return None + return _fail(REASON_TARGET_OUT_OF_RANGE) placeholder = self._placeholder(inventory, edit.slot) cname = self.fresh_name(working) lname = self.fresh_name( @@ -451,21 +519,21 @@ def apply( # The small-canonical-subtree precondition keeps the inverse # (restore old leaf comp + slot) expressible as the same action. if not (0 <= edit.stmt < len(working)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] if not target.has_list or len(target.children) != 1: - return None + return _fail(REASON_NOT_CANONICAL_SUBTREE) by_name = {s.name: s for s in working} leaf = by_name.get(target.children[0]) if leaf is None or leaf.has_list: - return None + return _fail(REASON_NOT_CANONICAL_SUBTREE) if not (0 <= edit.payload < len(self.components)): - return None + return _fail(REASON_PAYLOAD_OUT_OF_RANGE) leaf_comp = self.components[edit.payload] if leaf_comp not in LEAF_COMPONENTS: - return None + return _fail(REASON_PAYLOAD_NOT_LEAF) if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): - return None + return _fail(REASON_SLOT_OUT_OF_RANGE) placeholder = self._placeholder(inventory, edit.slot) leaf.comp = leaf_comp leaf.rest = json.dumps(placeholder, ensure_ascii=False) @@ -476,15 +544,15 @@ def apply( # Preconditions: MAX_STMTS bound, payload indexes V05_TEMPLATES. # Inverse: REMOVE (pack statements are unreferenced). if len(working) >= MAX_STMTS: - return None + return _fail(REASON_MAX_STMTS) if not (0 <= edit.payload < len(V05_TEMPLATES)): - return None + return _fail(REASON_PAYLOAD_OUT_OF_RANGE) comp, args = V05_TEMPLATES[edit.payload] candidate = Statement(self.fresh_v05_name(working, comp), comp, [], args, False) try: validate_output(candidate.render(), kind="statement") except Exception: # noqa: BLE001 - return None + return _fail(REASON_STATEMENT_INVALID) working.append(candidate) elif edit.action == ACTION_REPLACE_STATEMENT: # Swap one canonical V0.5 statement for another template. @@ -492,20 +560,20 @@ def apply( # (so the inverse — restore the old template — is expressible), # payload indexes V05_TEMPLATES, and the swap is a real change. if not (0 <= edit.stmt < len(working)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] if v05_template_index(target) is None: - return None + return _fail(REASON_TARGET_NOT_CANONICAL) if not (0 <= edit.payload < len(V05_TEMPLATES)): - return None + return _fail(REASON_PAYLOAD_OUT_OF_RANGE) comp, args = V05_TEMPLATES[edit.payload] if target.comp == comp and target.rest.strip() == args: - return None + return _fail(REASON_NO_OP) candidate = Statement(target.name, comp, [], args, False) try: validate_output(candidate.render(), kind="statement") except Exception: # noqa: BLE001 - return None + return _fail(REASON_STATEMENT_INVALID) working[edit.stmt] = candidate elif edit.action == ACTION_BIND_PLACEHOLDER: # Transactional declaration-plus-reference: (re)bind a leaf's slot @@ -513,23 +581,23 @@ def apply( # (non-root, non-container, leaf component), slot in inventory. # Inverse: BIND_PLACEHOLDER with the old slot index. if not (0 <= edit.stmt < len(working)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] if target.has_list or target.name == "root": - return None + return _fail(REASON_NOT_BINDABLE) if target.comp not in LEAF_COMPONENTS: - return None + return _fail(REASON_NOT_LEAF_COMPONENT) if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): - return None + return _fail(REASON_SLOT_OUT_OF_RANGE) placeholder = self._placeholder(inventory, edit.slot) target.rest = json.dumps(placeholder, ensure_ascii=False) else: - return None + return _fail(REASON_UNKNOWN_ACTION) if pre_validate is not None and not pre_validate(working): - return None + return _fail(REASON_PRE_VALIDATE_REJECTED) rendered = render_statements(working) if not _is_valid(rendered): - return None + return _fail(REASON_INVALID_RESULT) return working def sample_mutation( @@ -537,24 +605,45 @@ def sample_mutation( statements: list[Statement], inventory: list[str], rng: random.Random, + inverse_action_weights: dict[int, float] | None = None, ) -> tuple[list[Statement], Edit] | None: """One random validity-preserving mutation and the *inverse* edit - (the supervised repair step) — Kapur's forward process.""" + (the supervised repair step) — Kapur's forward process. + + ``inverse_action_weights`` (SLM-310) optionally declares a target + distribution over the *inverse* (repair) action: the inverse action + is sampled from the declared weights and mapped back to a mutation + kind that can realize it (``INVERSE_TO_MUTATION_KINDS``). Inverse + actions with no realizing mutation kind are skipped via the normal + retry loop. None = historical uniform-over-mutation-kinds behavior + (checkpoint / seed parity). + """ for _ in range(12): - kind = rng.choice( - ( - ACTION_REPLACE, - ACTION_ADD, - ACTION_REMOVE, - ACTION_ADD_CONTAINER, - ACTION_REMOVE_CONTAINER, - ACTION_INSERT_SUBTREE, - ACTION_REPLACE_SUBTREE, - ACTION_INSERT_STATEMENT, - ACTION_REPLACE_STATEMENT, - ACTION_BIND_PLACEHOLDER, + if inverse_action_weights: + population = sorted(inverse_action_weights) + inverse_action = rng.choices( + population, + weights=[inverse_action_weights[a] for a in population], + )[0] + kinds = INVERSE_TO_MUTATION_KINDS.get(inverse_action, ()) + if not kinds: + continue + kind = rng.choice(kinds) + else: + kind = rng.choice( + ( + ACTION_REPLACE, + ACTION_ADD, + ACTION_REMOVE, + ACTION_ADD_CONTAINER, + ACTION_REMOVE_CONTAINER, + ACTION_INSERT_SUBTREE, + ACTION_REPLACE_SUBTREE, + ACTION_INSERT_STATEMENT, + ACTION_REPLACE_STATEMENT, + ACTION_BIND_PLACEHOLDER, + ) ) - ) if kind == ACTION_REPLACE: idx = rng.randrange(len(statements)) stmt = statements[idx] @@ -884,6 +973,19 @@ class TreeEditDiffusionConfig: # (see ``from_checkpoint``) for behavior parity. value_label_mode: str = "bounded_distance" pairwise_progress_margin: float = 0.1 + # SLM-310 (LAR2-03): declared inverse-action distribution for the + # corruption sampler, mapping action NAME (``ACTION_NAMES``, e.g. "ADD") + # to a non-negative weight. Reweights sample_mutation's forward-noise + # choice toward mutation kinds whose inverse edit matches the declared + # distribution (e.g. ADD-balanced supervision); the gold corpus is never + # touched. None (default) = historical uniform behavior. + corruption_action_distribution: dict[str, float] | None = None + # SLM-310: STOP-slot accounting during decode. "legacy" (default, + # historical): every enumerated STOP proposal consumes an expand_per_state + # slot even when its frozen candidate is dropped as a duplicate. + # "corrected": STOP consumes a slot only when its frozen candidate is + # actually retained on the beam. + stop_slot_accounting: str = "legacy" # SLM-308: oracle depth/budget used for training-time value labels. The @@ -1085,6 +1187,28 @@ def _distance_label(self, statements, target, inventory, witness: int): upper_bound_witness=witness, ) + def _inverse_action_weights(self) -> dict[int, float] | None: + """SLM-310 corruption sampler: declared inverse-action weights keyed + by action id, or None for the historical uniform sampler.""" + declared = self.config.corruption_action_distribution + if not declared: + return None + weights: dict[int, float] = {} + for name, weight in declared.items(): + if name not in ACTION_IDS: + raise ValueError( + f"corruption_action_distribution names unknown action {name!r} " + f"(known: {sorted(ACTION_IDS)})" + ) + if weight < 0: + raise ValueError( + f"corruption_action_distribution weight for {name!r} is negative" + ) + weights[ACTION_IDS[name]] = float(weight) + if not any(weights.values()): + raise ValueError("corruption_action_distribution has no positive weight") + return weights + def forward(self, batch: list[ExampleRecord]) -> float: return float(self.training_loss(batch).detach().cpu()) @@ -1093,6 +1217,11 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: raise ValueError( f"unknown value_label_mode {self.config.value_label_mode!r}" ) + if self.config.stop_slot_accounting not in {"legacy", "corrected"}: + raise ValueError( + f"unknown stop_slot_accounting {self.config.stop_slot_accounting!r}" + ) + inverse_weights = self._inverse_action_weights() bounded_mode = self.config.value_label_mode == "bounded_distance" prompts: list[str] = [] states: list[str] = [] @@ -1133,7 +1262,10 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: inverse: Edit | None = None applied = 0 for _ in range(k): - step = self.space.sample_mutation(current, inventory, self._rng) + step = self.space.sample_mutation( + current, inventory, self._rng, + inverse_action_weights=inverse_weights, + ) if step is None: break prev = current @@ -1391,8 +1523,24 @@ def _decode_one( seed = self._seed_state(inventory) if seed is None: return "", {"failure": "no_valid_seed"} + if self.config.stop_slot_accounting not in {"legacy", "corrected"}: + raise ValueError( + f"unknown stop_slot_accounting {self.config.stop_slot_accounting!r}" + ) beam: list[tuple[float, list[Statement], bool]] = [(0.0, seed, False)] evidence: dict[str, Any] = {"steps": 0, "expansions": 0, "kind": "tree_edit"} + # SLM-310: per-proposal reason-coded applicability telemetry. Every + # enumerated candidate the decode loop visits is recorded in + # deterministic (score-sorted) enumeration order with its head + # log-prob factor score, whether TreeEditSpace.apply accepts it, the + # rejection reason code when rejected, whether it consumed + # expand_per_state budget, and whether it was retained on the beam. + proposals: list[dict[str, Any]] = [] + # Expanded (live) states per step, so downstream audits can re-check + # full edit-space applicability on exactly the states decode visited. + states_log: list[dict[str, Any]] = [] + verifier_calls = 0 + corrected_stop = self.config.stop_slot_accounting == "corrected" for _ in range(self.config.max_search_steps): live = [entry for entry in beam if not entry[2]] if not live: @@ -1411,32 +1559,76 @@ def _decode_one( render_statements(s) for _, s, frozen in next_beam if frozen } for row, (_, statements, _) in enumerate(live): + states_log.append( + { + "step": evidence["steps"], + "beam_row": row, + "source": render_statements(statements), + } + ) candidates = self._enumerate_edits( out, row, len(statements), len(inventory) ) expanded = 0 - for _, edit in candidates: + for rank, (score, edit) in enumerate(candidates): if expanded >= self.config.expand_per_state: break + record: dict[str, Any] = { + "step": evidence["steps"], + "beam_row": row, + "rank": rank, + "action": edit.action, + "action_name": ACTION_NAMES[edit.action], + "score": float(score), + } + proposals.append(record) if edit.action == ACTION_STOP: text = render_statements(statements) - if text not in seen: + retained = text not in seen + record["applicable"] = True + record["rejection_reason"] = ( + None if retained else REASON_DUPLICATE_STATE + ) + record["selected"] = retained + if retained: seen.add(text) next_beam.append( (float(out["value"][row]), statements, True) ) - expanded += 1 + # STOP-slot accounting arms (SLM-310): legacy consumes + # a slot for every STOP proposal; corrected consumes + # one only when the frozen candidate is retained. + consumed = retained if corrected_stop else True + record["consumed_budget"] = consumed + if consumed: + expanded += 1 continue - child = self.space.apply(statements, edit, inventory) + reason: list[str] = [] + child = self.space.apply( + statements, edit, inventory, reason=reason + ) + verifier_calls += 1 if child is None: + record["applicable"] = False + record["rejection_reason"] = reason[0] + record["selected"] = False + record["consumed_budget"] = False continue text = render_statements(child) if text in seen: + record["applicable"] = True + record["rejection_reason"] = REASON_DUPLICATE_STATE + record["selected"] = False + record["consumed_budget"] = False continue seen.add(text) next_beam.append((float(out["value"][row]), child, False)) expanded += 1 evidence["expansions"] += 1 + record["applicable"] = True + record["rejection_reason"] = None + record["selected"] = True + record["consumed_budget"] = True if not next_beam: break # Re-score unfrozen children by the value head (Kapur's search @@ -1466,6 +1658,21 @@ def _decode_one( best = max(beam, key=lambda entry: entry[0]) evidence["value"] = float(best[0]) evidence["frozen"] = bool(best[2]) + evidence["proposals"] = proposals + evidence["states"] = states_log + evidence["verifier_calls"] = verifier_calls + evidence["proposal_summary"] = { + "visited": len(proposals), + "applicable": sum(1 for p in proposals if p["applicable"]), + "selected": sum(1 for p in proposals if p["selected"]), + "consumed_budget": sum(1 for p in proposals if p["consumed_budget"]), + "dead": sum( + 1 + for p in proposals + if not p["applicable"] + or p["rejection_reason"] == REASON_DUPLICATE_STATE + ), + } return render_statements(best[1]), evidence def generate_batch_requests(self, requests: list[GenerationRequest]) -> list[str]: diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 7a3a7c3bd..015d3f848 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -4480,7 +4480,7 @@ ] }, "harness.experiments.slm299_edit_reachability": { - "version": "v3", + "version": "v4", "kind": "harness", "paths": [ "src/slm_training/harnesses/experiments/slm299_edit_reachability.py", @@ -4494,6 +4494,11 @@ "tests/test_models/test_tree_edit_diffusion.py" ], "history": [ + { + "version": "v4", + "date": "2026-07-24", + "note": "SLM-310: tree_edit_diffusion gains reason-coded apply rejections, per-proposal decode telemetry (applicability/rejection/budget), corruption_action_distribution + stop_slot_accounting config knobs (defaults = historical behavior, checkpoint parity); test_tree_edit_diffusion unchanged semantics" + }, { "version": "v3", "date": "2026-07-24", @@ -4530,25 +4535,6 @@ } ] }, - "harness.experiments.slm301_prompt_observability": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm301_prompt_observability.py", - "scripts/run_slm301_prompt_observability.py", - "tests/test_harnesses/experiments/test_slm301_prompt_observability.py", - "docs/design/iter-slm301-prompt-observability-20260724.json", - "docs/design/iter-slm301-prompt-observability-20260724.md", - "docs/design/slm301-slot-observability-audit-20260724.json" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-301 LAR1-04 prompt-observability arms (as-is vs +inventory) with phase-0 coverage classification, AR/X22 baselines, paired predeclared-delta report" - } - ] - }, "harness.experiments.slm303_decode_budget_audit": { "version": "v1", "kind": "harness", @@ -6312,7 +6298,7 @@ { "version": "v223", "date": "2026-07-22", - "note": "Derive component-array boundaries, nonempty typed children, available-content closure, typed binder compatibility, and acyclic references from schema so constrained decode cannot enter impossible continuations" + "note": "Derive component-array boundaries, nonempty typed children, available-content closure, typed binder compatibility, and acyclic references from schema so constrained decode cannot enter impossible continuations" }, { "version": "v222", @@ -7837,6 +7823,24 @@ "note": "initial registration; byte-preserving archived harness replay records" } ] + }, + "harness.experiments.slm310_action_alignment": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm310_action_alignment.py", + "scripts/run_slm310_action_alignment.py", + "tests/test_harnesses/experiments/test_slm310_action_alignment.py", + "docs/design/iter-slm310-action-alignment-20260724.json", + "docs/design/iter-slm310-action-alignment-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-310 LAR2-03 action alignment: reason-coded proposal telemetry, train-target vs decode-demand distribution audit, declared inverse-action corruption sampler, legacy/corrected STOP-slot accounting arms, matched 2x2 fixture experiment with preregistered thresholds" + } + ] } } } diff --git a/tests/test_harnesses/experiments/test_slm310_action_alignment.py b/tests/test_harnesses/experiments/test_slm310_action_alignment.py new file mode 100644 index 000000000..7dc01e9b8 --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm310_action_alignment.py @@ -0,0 +1,405 @@ +"""SLM-310 (LAR2-03): action-alignment invariants for the X22 tree-edit model.""" + +from __future__ import annotations + +import json +import random + +import pytest + +torch = pytest.importorskip("torch") + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm310_action_alignment import ( + action_calibration, + decode_demand_distribution, + run_distribution_audit, + training_target_distribution, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_ADD, + ACTION_BIND_PLACEHOLDER, + ACTION_IDS, + ACTION_REMOVE, + ACTION_STOP, + Edit, + REASON_DUPLICATE_STATE, + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + TreeEditSpace, + parse_statements, +) +from scripts.run_slm310_action_alignment import CELLS + +PROGRAM = ( + 'root = Stack([inline_card, cta], "column")\n' + "inline_card = Card([title])\n" + 'title = TextContent(":hero.title")\n' + 'cta = Button(":cta.label")' +) +INVENTORY = [":hero.title", ":cta.label"] + +PROGRAMS = [ + PROGRAM, + 'root = Stack([t], "column")\nt = TextContent(":body")', + 'root = Form([i, s], "column")\ni = TextInput(":form.email")\ns = Button(":form.submit")', +] +INVENTORIES = [INVENTORY, [":body"], [":form.email", ":form.submit"]] + + +def _records() -> list[ExampleRecord]: + return [ + ExampleRecord( + id=f"fx_{i}", + prompt=f"fixture prompt {i}", + openui=openui, + placeholders=list(inv), + split="train", + ) + for i, (openui, inv) in enumerate(zip(PROGRAMS, INVENTORIES)) + ] + + +def _model(**overrides) -> TreeEditDiffusionModel: + base = dict( + d_model=32, + n_heads=4, + context_layers=1, + denoiser_layers=1, + beam_width=2, + expand_per_state=2, + max_search_steps=3, + seed=3, + value_label_mode="mutation_count", + ) + base.update(overrides) + cfg = TreeEditDiffusionConfig(**base) + torch.manual_seed(cfg.seed) # deterministic init across test order + return TreeEditDiffusionModel.from_records(_records(), config=cfg, device="cpu") + + +def _evidence(model: TreeEditDiffusionModel) -> list[dict]: + from slm_training.harnesses.model_build.plugin import GenerationRequest + + model.generate_batch_requests( + [ + GenerationRequest(prompt=r.prompt, slot_contract=tuple(r.placeholders)) + for r in _records() + ] + ) + return model.consume_generation_evidence() + + +# --- reason-coded applicability --------------------------------------------- + + +def test_apply_reports_reason_codes() -> None: + space = TreeEditSpace() + base = parse_statements(PROGRAM) + assert base is not None + # Out-of-range statement index. + reason: list[str] = [] + assert space.apply(base, Edit(ACTION_REMOVE, 99), INVENTORY, reason=reason) is None + assert reason == ["index_out_of_range"] + # Leaf/container mismatch on REPLACE (root is a container, leaf comp given). + reason = [] + leaf_comp = space.comp_index["TextContent"] + assert space.apply(base, Edit(1, 0, leaf_comp), INVENTORY, reason=reason) is None + assert reason == ["leaf_container_mismatch"] + # BIND_PLACEHOLDER on the root container. + reason = [] + assert ( + space.apply(base, Edit(ACTION_BIND_PLACEHOLDER, 0, slot=0), INVENTORY, reason=reason) + is None + ) + assert reason == ["not_bindable"] + # Acceptance appends nothing. + reason = [] + assert space.apply(base, Edit(ACTION_REMOVE, 3), INVENTORY, reason=reason) is not None + assert reason == [] + + +def test_proposal_telemetry_reason_coded_and_deterministic() -> None: + # Wide expansion budget so the decode loop visits inapplicable + # (dead) candidates, not only top-scoring applicable ones. + model = _model(expand_per_state=8, beam_width=3, max_search_steps=4) + first = _evidence(model) + second = _evidence(model) + # Deterministic: identical proposals across identical decodes. + assert [e.get("proposals") for e in first] == [e.get("proposals") for e in second] + saw_rejection = False + saw_reason_codes: set[str] = set() + for ev in first: + proposals = ev.get("proposals") + assert proposals, "every decode records proposal telemetry" + assert ev["verifier_calls"] == sum( + 1 for p in proposals if p["action"] != ACTION_STOP + ) + for p in proposals: + assert set(p) >= { + "step", + "beam_row", + "rank", + "action", + "action_name", + "score", + "applicable", + "rejection_reason", + "selected", + "consumed_budget", + } + # Deterministic order: ranks are strictly increasing per state. + if p["applicable"]: + assert p["rejection_reason"] in (None, REASON_DUPLICATE_STATE) + else: + assert isinstance(p["rejection_reason"], str) + saw_rejection = True + saw_reason_codes.add(p["rejection_reason"]) + # Budget accounting invariants: consumed implies applicable, and + # every selected proposal consumed budget. + if p["consumed_budget"]: + assert p["applicable"] + if p["selected"]: + assert p["consumed_budget"] + # Summary is internally consistent. + summary = ev["proposal_summary"] + assert summary["visited"] == len(proposals) + assert summary["applicable"] == sum(1 for p in proposals if p["applicable"]) + assert saw_rejection, "fixture decode must exercise dead candidates" + assert saw_reason_codes <= { + "index_out_of_range", + "no_op", + "leaf_container_mismatch", + "max_stmts", + "parent_or_comp_precondition", + "slot_out_of_range", + "not_removable", + "unreferenced_leaf", + "target_out_of_range", + "not_container", + "subtree_not_leaf_only", + "unreferenced_container", + "payload_out_of_range", + "payload_not_leaf", + "not_canonical_subtree", + "statement_invalid", + "target_not_canonical", + "not_bindable", + "not_leaf_component", + "unknown_action", + "pre_validate_rejected", + "invalid_result", + } + + +# --- distribution audit math --------------------------------------------------- + + +def test_training_target_distribution_math() -> None: + space = TreeEditSpace() + dist = training_target_distribution( + _records(), space, seed=5, samples=120, max_chain=3 + ) + assert dist["n_targets"] == sum(dist["counts"].values()) + assert abs(sum(dist["shares"].values()) - 1.0) < 1e-9 + assert dist["counts"]["STOP"] > 0 # stop_fraction branch exercised + # Deterministic. + again = training_target_distribution( + _records(), space, seed=5, samples=120, max_chain=3 + ) + assert again["counts"] == dist["counts"] + + +def test_decode_demand_audit_math() -> None: + model = _model() + demand = decode_demand_distribution(model, _records()) + overall = demand["overall"] + assert overall["visited_total"] > 0 + assert overall["verifier_calls"] > 0 + assert 0.0 <= overall["dead_candidate_rate"] <= 1.0 + assert abs(sum(overall["visited_shares"].values()) - 1.0) < 1e-6 + assert set(demand["by_source"]) == {r.id for r in _records()} + assert set(demand["by_suite"]) == {"train"} + # Dead total = visited - (no-reason proposals). + assert overall["dead_total"] <= overall["visited_total"] + # Calibration rows cover every action and MAD is the mean |gap|. + calib = action_calibration( + training_target_distribution(_records(), model.space, seed=5, samples=60), + overall, + ) + assert len(calib["per_action"]) == 11 + gaps = [abs(r["gap"]) for r in calib["per_action"].values()] + assert abs(calib["mean_abs_deviation"] - sum(gaps) / len(gaps)) < 1e-12 + # Full audit predicts via the preregistered rule and stays honest. + audit = run_distribution_audit(model, _records(), seed=5, samples=60) + pred = audit["loss_reweighting_prediction"] + assert isinstance(pred["predicted"], bool) + assert pred["applicable_add_recall"] == overall["applicable_add_recall"] + + +# --- corruption sampler --------------------------------------------------------- + + +def test_corruption_sampler_hits_declared_distribution() -> None: + space = TreeEditSpace() + statements = parse_statements(PROGRAM) + assert statements is not None + # Degenerate declared distribution: all weight on ADD inverses. + rng = random.Random(13) + inverses = [] + for _ in range(40): + step = space.sample_mutation( + statements, INVENTORY, rng, + inverse_action_weights={ACTION_ADD: 1.0}, + ) + if step is None: + continue + inverses.append(step[1].action) + assert inverses, "sampler must still produce mutations" + assert set(inverses) == {ACTION_ADD} + # Declared ADD-balanced distribution lands near the declared share. + declared = {ACTION_IDS[n]: w for n, w in { + "REPLACE": 1.0, "ADD": 4.0, "REMOVE": 1.0, "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, "INSERT_SUBTREE": 1.0, "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, "BIND_PLACEHOLDER": 1.0, + }.items()} + dist = training_target_distribution( + _records(), space, seed=7, samples=600, max_chain=3, + inverse_action_weights=declared, + ) + default = training_target_distribution( + _records(), space, seed=7, samples=600, max_chain=3 + ) + add_share = dist["shares"]["ADD"] + assert add_share > default["shares"]["ADD"] + 0.10 + # Declared ADD weight is 4/12 of inverse mass; with 20% STOP targets the + # realized share should sit near 0.8 * 4/12 ≈ 0.27 (loose tolerance for + # retry-loop fallback and REMOVE-mutation applicability). + assert abs(add_share - 0.8 * 4 / 12) < 0.12 + + +def test_corruption_sampler_leaves_gold_corpus_untouched() -> None: + records = _records() + before = [r.openui for r in records] + model = _model( + corruption_action_distribution={"ADD": 4.0, "REPLACE": 1.0, "REMOVE": 1.0} + ) + loss = model.training_loss(records) + assert torch.isfinite(loss) + assert [r.openui for r in records] == before + # Default off: config without the knob keeps the historical sampler. + assert _model()._inverse_action_weights() is None + # Unknown action names fail closed. + bad = _model(corruption_action_distribution={"NOT_AN_ACTION": 1.0}) + with pytest.raises(ValueError, match="unknown action"): + bad.training_loss(records) + + +# --- STOP-slot accounting arms --------------------------------------------------- + + +def test_stop_slot_accounting_arms_differ_exactly_as_specified() -> None: + legacy = _model(stop_slot_accounting="legacy") + corrected = _model(stop_slot_accounting="corrected") + leg_ev = _evidence(legacy) + cor_ev = _evidence(corrected) + # Deterministic per arm. + assert [e.get("proposals") for e in leg_ev] == [ + e.get("proposals") for e in _evidence(legacy) + ] + assert [e.get("proposals") for e in cor_ev] == [ + e.get("proposals") for e in _evidence(corrected) + ] + + def stop_stats(evs): + visited = consumed = retained = 0 + for ev in evs: + for p in ev.get("proposals", []): + if p["action"] != ACTION_STOP: + continue + visited += 1 + consumed += 1 if p["consumed_budget"] else 0 + retained += 1 if p["selected"] else 0 + return visited, consumed, retained + + lv, lc, lr = stop_stats(leg_ev) + cv, cc, cr = stop_stats(cor_ev) + assert lv > 0, "fixture decode must visit STOP proposals" + # Legacy: every STOP consumes a slot, retained or not. + assert lc == lv + # Corrected: STOP consumes a slot exactly when its frozen candidate is + # retained. + assert cc == cr + assert cc <= lc + # Unknown arm values fail closed. + bad = _model(stop_slot_accounting="bogus") + with pytest.raises(ValueError, match="stop_slot_accounting"): + _evidence(bad) + + +def test_lever_isolation_single_factor_per_arm() -> None: + def diff(x, y): + return [k for k in ("corruption", "stop") if CELLS[x][k] != CELLS[y][k]] + + assert diff("cell_b", "cell_a") == ["corruption"] + assert diff("cell_c", "cell_a") == ["stop"] + assert diff("cell_d", "cell_b") == ["stop"] + assert diff("cell_d", "cell_c") == ["corruption"] + + +# --- checkpoint parity ----------------------------------------------------------- + + +def test_checkpoint_parity_with_new_additive_fields(tmp_path) -> None: + records = _records() + model = _model(corruption_action_distribution={"ADD": 2.0, "REMOVE": 1.0}) + path = tmp_path / "ckpt.pt" + model.save(path) + loaded = TreeEditDiffusionModel.from_checkpoint(path, device="cpu") + assert loaded.config.corruption_action_distribution == { + "ADD": 2.0, + "REMOVE": 1.0, + } + assert loaded.config.stop_slot_accounting == "legacy" + # Simulate a pre-SLM-310 checkpoint: drop the new config keys entirely. + payload = torch.load(path, map_location="cpu", weights_only=False) + payload["config"].pop("corruption_action_distribution") + payload["config"].pop("stop_slot_accounting") + old_path = tmp_path / "ckpt_old.pt" + torch.save(payload, old_path) + (tmp_path / "ckpt_old.tokenizer.json").write_text( + (tmp_path / "ckpt.tokenizer.json").read_text(encoding="utf-8"), + encoding="utf-8", + ) + old = TreeEditDiffusionModel.from_checkpoint(old_path, device="cpu") + # Pre-field checkpoints get the historical defaults: uniform sampler, + # legacy STOP accounting. + assert old.config.corruption_action_distribution is None + assert old.config.stop_slot_accounting == "legacy" + # Decode parity: the old-loaded model reproduces the default model. + from slm_training.harnesses.model_build.plugin import GenerationRequest + + requests = [ + GenerationRequest(prompt=r.prompt, slot_contract=tuple(r.placeholders)) + for r in records + ] + default_model = _model() + assert old.generate_batch_requests(requests) == default_model.generate_batch_requests( + requests + ) + + +def test_metadata_not_polluted(tmp_path) -> None: + # Proposal telemetry stays on evidence; meta.json schema unchanged. + model = _model() + path = tmp_path / "ckpt.pt" + model.save(path) + meta = json.loads(path.with_suffix(".meta.json").read_text(encoding="utf-8")) + assert set(meta) == { + "kind", + "format_version", + "tokenizer", + "vocab_size", + "parameter_count", + "serialized_weight_bytes", + } From 0dd815193a4fccb41f229751393ed18a989718d7 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:15:09 -0500 Subject: [PATCH 07/10] SLM-312: seedward and on-policy state sources with matched mixtures - State-source harness: gold_only (existing corruption chain), seedward (offline oracle-guided walk from the seed toward gold, strictly distance-decreasing valid intermediates), on_policy (immutable content-addressed beam-trajectory snapshots: wrong states, verifier failures, abstentions); explicit provenance + gold-visibility policy per row; sha256 rows + tamper-evident manifest; fail-closed leakage guards (train vs held-out AST fingerprints). - Matched {gold_only, seedward, on_policy, mixed} arms with predeclared weights/caps, identical model/steps/optimizer/seeds; evaluated on held-out seed trajectories. Verdict: rejected per preregistered primary gate (beam regret improvement 0.0 < 0.05); secondary signal (value rank corr -0.258 -> +0.258 for non-gold arms) recorded as wiring evidence only. - 9 new tests; slm312 component v1. --- .../iter-slm312-state-sources-20260724.json | 468 +++++++ .../iter-slm312-state-sources-20260724.md | 40 + scripts/run_slm312_state_sources.py | 526 ++++++++ .../experiments/slm312_state_sources.py | 1150 +++++++++++++++++ src/slm_training/resources/versions.json | 18 + .../experiments/test_slm312_state_sources.py | 310 +++++ 6 files changed, 2512 insertions(+) create mode 100644 docs/design/iter-slm312-state-sources-20260724.json create mode 100644 docs/design/iter-slm312-state-sources-20260724.md create mode 100644 scripts/run_slm312_state_sources.py create mode 100644 src/slm_training/harnesses/experiments/slm312_state_sources.py create mode 100644 tests/test_harnesses/experiments/test_slm312_state_sources.py diff --git a/docs/design/iter-slm312-state-sources-20260724.json b/docs/design/iter-slm312-state-sources-20260724.json new file mode 100644 index 000000000..9f68bebfe --- /dev/null +++ b/docs/design/iter-slm312-state-sources-20260724.json @@ -0,0 +1,468 @@ +{ + "experiment": "slm312-state-sources", + "issue": "SLM-312", + "question": "Does closing the gold-corruption vs seed-rollout distribution gap (seedward + on-policy state sources) improve repair more than extra offline near-gold training at equal budget?", + "preregistered_thresholds": { + "beam_regret_improvement_min": 0.05, + "rank_correlation_degradation_max": 0.05, + "comparison": "best non-gold arm (seedward/on_policy/mixed) vs gold_only", + "verdict_rule": "distribution_gap_closed iff best non-gold arm improves beam regret >= 0.05 AND degrades rank correlation by <= 0.05, else rejected" + }, + "preregistered_arms": { + "gold_only": { + "gold_only": 1.0 + }, + "seedward": { + "seedward": 1.0 + }, + "on_policy": { + "on_policy": 1.0 + }, + "mixed": { + "gold_only": 0.34, + "seedward": 0.33, + "on_policy": 0.33 + } + }, + "config": { + "steps": 4, + "batch_size": 4, + "per_arm_budget": 24, + "seed": 0, + "n_train_records": 6, + "n_held_out_records": 2, + "suite_id": "slm312-fixture-v1", + "fixture_config_overrides": { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_search_steps": 8, + "beam_width": 3, + "expand_per_state": 3 + }, + "on_policy_max_states_per_record": 3, + "torch_num_threads": 1 + }, + "snapshot": { + "path": "outputs/slm312/state_snapshot.jsonl", + "manifest": { + "snapshot_schema": "slm312-state-snapshot/v1", + "snapshot_id": "slm312-state-sources-fixture", + "config": { + "suite_id": "slm312-fixture-v1", + "arms": { + "gold_only": { + "gold_only": 1.0 + }, + "seedward": { + "seedward": 1.0 + }, + "on_policy": { + "on_policy": 1.0 + }, + "mixed": { + "gold_only": 0.34, + "seedward": 0.33, + "on_policy": 0.33 + } + }, + "per_arm_budget": 24, + "steps": 4, + "batch_size": 4, + "seed": 0, + "held_out_fingerprints": [ + "8008d5caebe08c70e2ccc1a558d30294c8d3c09e334f402b56ef960caab7c034", + "b5d4c7824bfcd66ea60732a53d5de59068366a1cd92cc92b58a13c56c23b3dfe", + "d12577927c9c7c1e1afd0b86f726611be9a9e901a9269be41d0c19bf3eb22826", + "e18b0d82658d1117f34b7704c2537a4112e4c5f3ecd0cd7dc1c6ac7873e4cf9b", + "fb34d4793d23dcc327c1677c10bac6d3ca338308a9b96307371ea0cff6325e44" + ], + "dropped_leak_collisions": [ + { + "record_id": "fixture_01", + "source": "gold_only", + "kind": "near_gold", + "fingerprint": "d12577927c9c7c1e1afd0b86f726611be9a9e901a9269be41d0c19bf3eb22826", + "reason": "held_out_fingerprint_collision" + } + ] + }, + "n_rows": 43, + "row_sha256": [ + "e4c806299e446dd8c66e14488e9ea4f3e7f4e3df53edf8495226594fb646802a", + "45ca113c57daed51731f2316082d801e38557e7ef45ed75933296ef86aaf1a76", + "3a15f53adb710b15b13fca9073022baa2fc15b4c6901ea9e62be013f878c3100", + "6dd53388dde698e988e7d1f5a626e135dbc8e47b1aaf10de40bc2897ae80c43d", + "468851f57f3ac185df3ecb4061ebc56f790d321fd1b6fb561ec9d681e449184e", + "bfacacc8e17be88bf6025280ba01e1382165ed9fb1fe4770dfcf178a50c6fe77", + "bc5495c95a9cc193772dd0bfd66a6f46df6afeb34101b5443d22df1e2d13e58b", + "3cf4cd1f147b61fd819a956b459cc7993b7d2e1db9b45ca9d99c074d19bd6f69", + "541329e8a4f741bd5e2c165d00de59dd73f47992bcb42dd3639651b30f8abbf0", + "3253c262988b27d635204c7f825c684c52e2942d85da53093f2c8406da76d212", + "d19ce94773aa0fe4c672a214abe71aad6194052a2061315efc6ced5d666a92d0", + "e3824aa7db17af0dfa4e4c0069c464e5dce26cbf78c0c45364c101bb3c111777", + "d63ce3d52c05f2bc41081c2f4c7b280359f4666014f21df40e66b3445e4a9764", + "b7d17e0e6d789c47be762254e6359e2d30a13953d33c55d70d1e99b797a196a0", + "9d19b5dff62d5c37f7c514b6b9c222978114ccba63a5b40a7e9248f7f2ab7970", + "3e77f595f255fb0dfb3a77fa06406d75ce43c2bceb2a0f6d7c09bb1daac63ff3", + "7b7ec2845c0f96d71d06e1055c46dbf52473a7373e419589b2eb725470b216be", + "73082b08c6aadb30c269e27d473ef310d613bda05f75df462cc3b8ab24532c17", + "9b1c394db2db6f1f53f4aa5e7a65b84d6632603dced37ae9b00aa27fc71f56ff", + "b81f2c4618f162aa60d6000d9165eed6964e339372e1859ac18855ecfc744d1c", + "e16506ef496224edf4fcbda3f02e903f9d2417893821e5ab160125b9320ed02d", + "ce9e9b3ca03634fbae5909ab778176a36d57efe65b63d65eddff7b7a0c479e62", + "cab1651e429d37beda52a0945f2b69ff702e611b1e047bf7e6cf0a7137dc8ab1", + "2d1058b70c7630ff6be864d4c2ea9be99d720c2d110b33fd50d97292e274b056", + "e0577d5ee84d94ac7d166efa259f77f552f26cbb16a3d4b06e33aa172782426c", + "3444ac129fdbd1cc80c5921dc41d0e0952358baad8d1d9d34cc5ad2bcc033df9", + "46e4a474ca7130324949b26bcb889f3d598fe7f44b6f232e23f0bfaac3577dab", + "c4db64855ca8c7335df1b4097a2675b78740b141c97c656cab153f308c412072", + "c7e474e54c16bfc6122d6e975a920d1d3a57ac722b367266b9a43c5788c38739", + "4608be6d174bfba6e0423f5924e94a57907a94d009d7ecb23463ac67a158c64b", + "5bccd49029d79b60b40fa6097ab4feb734cc97d4dc1dca8f93286f332d7dc67c", + "c71f33b629b18e4f28b512f0102de4103a4a5240dc0f9bac39fc559346742ee5", + "d2c75520ce823ad8b70e23df7a6b679a398fe8aad1bbf1eb44412d791ab638fe", + "fb47281652480bf7210aeb4ed2c4d7dae4892d091ee9fd13eb76b142a43851e6", + "515d2607be5696e82e109188d4e7c30bd88d51384febf38a465ce223b74c4705", + "9a42b62454c5a568b47d287a25697c82e569d852a6bf2baee472f46bac2d0cd3", + "51c5d01b01be5da4be2a5ac3b42915dfcea0426bc64dade805dd1b391e8fc2c7", + "7f328c9728482b7b2df76810f15fed1f2569142e37f541962b624242120be1ed", + "3798721b3c9c21b3ecba164943a9db6058db8edfffc8f8775cf1b05fdc636fee", + "b7f1430f244943bda8efa33753e0630c44cf1e4eae14b0cf5a3ffe01a928ef10", + "084b655bca1975150f50af36ec63e372cad6b6b82baeb5b29f4412e381a8c1e2", + "54a2405447bc92161495ac9937b925a6b97e82e74d380a3efae04566e8f49985", + "e71f2adb4ac78b3e9efdfbbc628c14482dd6246c494ed495538c1f33b8809654" + ], + "manifest_sha256": "1c5fc80b721b12f4c6541749cade9dd05d926051b682217073d86fa26973fe01" + }, + "coverage": { + "rows": 43, + "unique_fingerprints": 37, + "duplicate_rate": 0.13953488372093026, + "per_source": { + "gold_only": { + "rows": 17, + "unique_fingerprints": 17 + }, + "on_policy": { + "rows": 18, + "unique_fingerprints": 18 + }, + "seedward": { + "rows": 8, + "unique_fingerprints": 8 + } + } + } + }, + "oracle_cost": { + "n_states": 6, + "n_unknown": 2, + "unknown_coverage": 0.3333333333333333, + "n_measurable": 4, + "wall_ms_total": 8382.73887700052, + "ms_per_state": 1397.1231461667533, + "cache": { + "label_hits": 0, + "label_misses": 6, + "map_builds": 2, + "target_maps": 2, + "labels": 6, + "map_nodes": 16 + } + }, + "arms": { + "gold_only": { + "n_rows": 17, + "coverage": { + "rows": 17, + "unique_fingerprints": 17, + "duplicate_rate": 0.0, + "per_source": { + "gold_only": { + "rows": 17, + "unique_fingerprints": 17 + } + } + }, + "metrics": { + "n_measurable": 4, + "rank_correlation": -0.2581988897471611, + "concordance": 0.3333333333333333, + "brier": 0.03396308228040912, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 1, + "mean_predicted_value": 0.6047580242156982, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 3, + "mean_predicted_value": 0.6053926746050516, + "mean_target": 0.75 + } + ], + "beam_regret_mean": 0.0, + "beam_regret_n": 1, + "beam_regret_excluded": 3, + "by_origin": {} + }, + "valid_final": { + "n": 2, + "valid_final_rate": 1.0, + "finals": [ + { + "record_id": "fixture_06", + "valid": true, + "steps": 4, + "verifier_calls": 698 + }, + { + "record_id": "fixture_07", + "valid": true, + "steps": 5, + "verifier_calls": 710 + } + ] + }, + "training_metrics": { + "action": 2.2585463523864746, + "stmt": 2.9041709899902344, + "comp": 1.8313002586364746, + "slot": 3.4643394947052, + "value": 0.07977510988712311, + "pairwise_progress": 0.0953509584069252, + "value_unknown_excluded": 2.0, + "skipped": 0.0, + "rows": 4.0 + } + }, + "mixed": { + "n_rows": 24, + "coverage": { + "rows": 24, + "unique_fingerprints": 24, + "duplicate_rate": 0.0, + "per_source": { + "gold_only": { + "rows": 8, + "unique_fingerprints": 8 + }, + "on_policy": { + "rows": 8, + "unique_fingerprints": 8 + }, + "seedward": { + "rows": 8, + "unique_fingerprints": 8 + } + } + }, + "metrics": { + "n_measurable": 4, + "rank_correlation": 0.2581988897471611, + "concordance": 0.6666666666666666, + "brier": 0.006841517524175167, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 1, + "mean_predicted_value": 0.7193946242332458, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 3, + "mean_predicted_value": 0.7179443836212158, + "mean_target": 0.75 + } + ], + "beam_regret_mean": 0.0, + "beam_regret_n": 1, + "beam_regret_excluded": 3, + "by_origin": {} + }, + "valid_final": { + "n": 2, + "valid_final_rate": 1.0, + "finals": [ + { + "record_id": "fixture_06", + "valid": true, + "steps": 8, + "verifier_calls": 887 + }, + { + "record_id": "fixture_07", + "valid": true, + "steps": 8, + "verifier_calls": 1816 + } + ] + }, + "training_metrics": { + "action": 2.153684139251709, + "stmt": 2.635714054107666, + "comp": 1.7767260074615479, + "slot": 1.1594464778900146, + "value": 0.0398101769387722, + "pairwise_progress": 0.1034640446305275, + "value_unknown_excluded": 3.0, + "skipped": 0.0, + "rows": 4.0 + } + }, + "on_policy": { + "n_rows": 18, + "coverage": { + "rows": 18, + "unique_fingerprints": 18, + "duplicate_rate": 0.0, + "per_source": { + "on_policy": { + "rows": 18, + "unique_fingerprints": 18 + } + } + }, + "metrics": { + "n_measurable": 4, + "rank_correlation": 0.2581988897471611, + "concordance": 0.6666666666666666, + "brier": 0.014381557280683133, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 1, + "mean_predicted_value": 0.6760445833206177, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 3, + "mean_predicted_value": 0.6730026205380758, + "mean_target": 0.75 + } + ], + "beam_regret_mean": 0.0, + "beam_regret_n": 1, + "beam_regret_excluded": 3, + "by_origin": {} + }, + "valid_final": { + "n": 2, + "valid_final_rate": 1.0, + "finals": [ + { + "record_id": "fixture_06", + "valid": true, + "steps": 8, + "verifier_calls": 2598 + }, + { + "record_id": "fixture_07", + "valid": true, + "steps": 8, + "verifier_calls": 2598 + } + ] + }, + "training_metrics": { + "action": 2.603577136993408, + "stmt": 3.4750404357910156, + "value": 0.08445368707180023, + "pairwise_progress": 0.1012253537774086, + "value_unknown_excluded": 0.0, + "skipped": 0.0, + "rows": 4.0 + } + }, + "seedward": { + "n_rows": 8, + "coverage": { + "rows": 8, + "unique_fingerprints": 8, + "duplicate_rate": 0.0, + "per_source": { + "seedward": { + "rows": 8, + "unique_fingerprints": 8 + } + } + }, + "metrics": { + "n_measurable": 4, + "rank_correlation": 0.2581988897471611, + "concordance": 0.6666666666666666, + "brier": 0.03013388784179849, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 1, + "mean_predicted_value": 0.6180281043052673, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 3, + "mean_predicted_value": 0.6153192520141602, + "mean_target": 0.75 + } + ], + "beam_regret_mean": null, + "beam_regret_n": 0, + "beam_regret_excluded": 4, + "by_origin": {} + }, + "valid_final": { + "n": 2, + "valid_final_rate": 1.0, + "finals": [ + { + "record_id": "fixture_06", + "valid": true, + "steps": 2, + "verifier_calls": 11 + }, + { + "record_id": "fixture_07", + "valid": true, + "steps": 2, + "verifier_calls": 11 + } + ] + }, + "training_metrics": { + "action": 0.6870732307434082, + "value": 0.16490422189235687, + "value_unknown_excluded": 0.0, + "skipped": 0.0, + "rows": 4.0 + } + } + }, + "comparison": { + "best_non_gold_arm": "on_policy", + "beam_regret_improvement": 0.0, + "rank_correlation_degradation": -0.5163977794943222 + }, + "threshold_checks": { + "beam_regret_ok": false, + "rank_ok": true + }, + "verdict": "rejected", + "wall_seconds": 84.94782193800347, + "honesty": "Fixture-scale matched arms over immutable content-addressed state snapshots; evaluation uses held-out seed trajectories only (fail-closed leak guard ran before training); UNKNOWN/unbounded oracle labels are excluded and counted, never coerced. A fixture verdict is wiring/state-source evidence, not a production ship claim.", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "d8f35563d231e3b404b0fb0e213d68aee1b00a9d", + "code_dirty": true, + "components": { + "harness.experiments.slm312_state_sources": "v1", + "harness.experiments.slm299_edit_reachability": "v4" + }, + "stamped_at": "2026-07-24T23:08:24.625582+00:00" + } +} diff --git a/docs/design/iter-slm312-state-sources-20260724.md b/docs/design/iter-slm312-state-sources-20260724.md new file mode 100644 index 000000000..e1170bb48 --- /dev/null +++ b/docs/design/iter-slm312-state-sources-20260724.md @@ -0,0 +1,40 @@ +# SLM-312 (LAR2-04): seedward + on-policy state sources vs near-gold + +**Verdict: `rejected`** (fixture-scale matched arms; not a ship claim) + +## Preregistered (locked before results) + +- arms/weights: `{"gold_only": {"gold_only": 1.0}, "seedward": {"seedward": 1.0}, "on_policy": {"on_policy": 1.0}, "mixed": {"gold_only": 0.34, "seedward": 0.33, "on_policy": 0.33}}` +- per-arm budget: 24 rows; identical model/steps/optimizer/seeds/evaluator +- beam regret improvement >= 0.05 +- rank correlation degradation <= 0.05 +- rule: distribution_gap_closed iff best non-gold arm improves beam regret >= 0.05 AND degrades rank correlation by <= 0.05, else rejected + +## Headline (held-out seed trajectories) + +| arm | rows | rank corr | concordance | Brier | beam regret | valid-final | +| --- | --- | --- | --- | --- | --- | --- | +| gold_only | 17 | -0.2582 | 0.3333 | 0.0340 | 0.0000 | 1.0000 | +| mixed | 24 | 0.2582 | 0.6667 | 0.0068 | 0.0000 | 1.0000 | +| on_policy | 18 | 0.2582 | 0.6667 | 0.0144 | 0.0000 | 1.0000 | +| seedward | 8 | 0.2582 | 0.6667 | 0.0301 | n/a | 1.0000 | + +Best non-gold arm: **on_policy** — beam regret improvement 0.0000, rank correlation degradation -0.5164. + +## State-space coverage by source (full snapshot) + +| source | rows | unique fingerprints | +| --- | --- | --- | +| gold_only | 17 | 17 | +| on_policy | 18 | 18 | +| seedward | 8 | 8 | + +Snapshot duplicate rate 0.140; 37/43 unique states. Manifest `1c5fc80b721b12f4…` at `outputs/slm312/state_snapshot.jsonl`. + +## Coverage + oracle cost + +- UNKNOWN coverage 0.333 (2/6), measurable 4, oracle 1397.1 ms/state + +## Honesty + +Fixture-scale matched arms over immutable content-addressed state snapshots; evaluation uses held-out seed trajectories only (fail-closed leak guard ran before training); UNKNOWN/unbounded oracle labels are excluded and counted, never coerced. A fixture verdict is wiring/state-source evidence, not a production ship claim. diff --git a/scripts/run_slm312_state_sources.py b/scripts/run_slm312_state_sources.py new file mode 100644 index 000000000..fc7386042 --- /dev/null +++ b/scripts/run_slm312_state_sources.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python3 +"""SLM-312 (LAR2-04): matched state-source mixture arms {gold_only, seedward, on_policy, mixed}. + +Question: does closing the gold-corruption vs seed-rollout distribution gap +(training on seedward and/or on-policy valid states, oracle-labeled) improve +repair more than extra offline near-gold training, at equal model / steps / +optimizer / evaluator / seed budget? + +Preregistered (written into the output payload BEFORE any result): + +- arms + mixture weights + acquisition caps: ``PREREGISTERED_ARMS`` and + ``--per-arm-budget`` (locked before training; deviations append-only); +- primary: best non-gold arm must improve beam regret vs ``gold_only`` by + **>= 0.05** (distance units) AND must not degrade value rank correlation + by more than **0.05**; +- verdict ``distribution_gap_closed`` iff both hold, else ``rejected`` — + honestly computed from the measured arms, never narrated. + +Arms share the tiny fixture corpus, model init seed, optimizer budget, batch +order, and evaluator; they differ ONLY in which state source feeds +``state_supervision_loss``. Evaluation runs on HELD-OUT SEED TRAJECTORIES +(random walks from the decode seed over held-out records — never acquired +states): valid-final decode rate, value calibration (rank correlation, +concordance, Brier, per-bin), beam regret vs oracle-best (SLM-308 +machinery), AST-fingerprint duplicate rate + state-space coverage per +source. Leak guards are fail-closed: any train/held-out fingerprint overlap +aborts the run before training. + +Writes ``docs/design/iter-slm312-state-sources-20260724.{json,md}`` and the +immutable snapshot under ``outputs/slm312/``. + +Example: + python -m scripts.run_slm312_state_sources --steps 4 +""" + +from __future__ import annotations + +import argparse +import json +import random +import time +from pathlib import Path + +import torch + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm308_distance_oracle import ( + clear_caches, +) +from slm_training.harnesses.experiments.slm312_state_sources import ( + EXPERIMENT_ID, + PREREGISTERED_ARMS, + SEED_SOURCE, + StateSnapshotV1, + acquire_gold_only, + acquire_on_policy, + acquire_seedward, + assert_no_leakage, + build_arm_rows, + source_coverage, + source_fingerprint, + state_supervision_loss, + drop_leaked_rows, +) +from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + TreeEditSpace, + _is_valid, + parse_statements, + render_statements, +) +from slm_training.versioning import build_version_stamp +from scripts.run_slm308_distance_value import ( + FIXTURE_PROGRAMS, + label_eval_states, + score_arm, +) + +DEFAULT_JSON_OUT = Path("docs/design/iter-slm312-state-sources-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm312-state-sources-20260724.md") +DEFAULT_SNAPSHOT_OUT = Path("outputs/slm312/state_snapshot.jsonl") + +SUITE_ID = "slm312-fixture-v1" +N_TRAIN_RECORDS = 6 # last len(FIXTURE_PROGRAMS) - N_TRAIN_RECORDS are held out + +# Tiny fixture model/decode config — identical across ALL arms and declared in +# the payload; it only bounds fixture wall-clock (the run cap), never differs +# by arm. +FIXTURE_CONFIG_OVERRIDES = { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_search_steps": 8, + "beam_width": 3, + "expand_per_state": 3, +} +ON_POLICY_MAX_STATES_PER_RECORD = 3 + +# PREREGISTERED improvement thresholds (locked before any run; do not edit +# after outcomes are visible — deviations are append-only and exploratory). +PREREGISTERED_THRESHOLDS = { + "beam_regret_improvement_min": 0.05, + "rank_correlation_degradation_max": 0.05, + "comparison": "best non-gold arm (seedward/on_policy/mixed) vs gold_only", + "verdict_rule": ( + "distribution_gap_closed iff best non-gold arm improves beam regret " + ">= 0.05 AND degrades rank correlation by <= 0.05, else rejected" + ), +} + + +def build_records() -> tuple[list[ExampleRecord], list[ExampleRecord]]: + records = [ + ExampleRecord( + id=f"fixture_{i:02d}", + prompt=prompt, + openui=openui, + placeholders=list(placeholders), + split="train" if i < N_TRAIN_RECORDS else "held_out", + ) + for i, (prompt, openui, placeholders) in enumerate(FIXTURE_PROGRAMS) + ] + return records[:N_TRAIN_RECORDS], records[N_TRAIN_RECORDS:] + + +def build_heldout_eval_states( + held_out: list[ExampleRecord], space: TreeEditSpace, *, seed: int = 31337 +) -> list[dict]: + """Held-out seed-trajectory eval states: random walks from the decode + seed over HELD-OUT records only (distinct rng stream from training).""" + states: list[dict] = [] + seed_statements = parse_statements(SEED_SOURCE) + assert seed_statements is not None + for idx, record in enumerate(held_out): + gold = parse_statements(record.openui) + assert gold is not None + inventory = [p if p.startswith(":") else f":{p}" for p in record.placeholders] + rng = random.Random(seed + idx) + # The bare decode seed is the origin of every seed trajectory; it is + # almost always oracle-measurable (small EXACT distance to gold). + states.append( + { + "record_idx": idx, + "origin": "heldout_seed_trajectory", + "witness": None, + "statements": seed_statements, + "target": gold, + "inventory": inventory, + } + ) + # Walk depths 1-2: deeper walks are almost always oracle-UNKNOWN at + # the shallow SLM-308 budgets and only cost eval time. + for k in (1, 2): + current = seed_statements + applied = 0 + for _ in range(k): + step = space.sample_mutation(current, inventory, rng) + if step is None: + break + current, _inverse = step + applied += 1 + if applied == 0: + continue + states.append( + { + "record_idx": idx, + "origin": "heldout_seed_trajectory", + "witness": None, + "statements": current, + "target": gold, + "inventory": inventory, + } + ) + return states + + +def train_arm( + records: list[ExampleRecord], + arm_rows: list, + *, + steps: int, + batch_size: int, + seed: int, +) -> TreeEditDiffusionModel: + """One arm: identical model init / optimizer / batch order; only the + state source differs (loss consumes the arm's snapshot rows).""" + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + seed=seed, + value_label_mode="bounded_distance", + context_backend="scratch", + max_chain=3, + **FIXTURE_CONFIG_OVERRIDES, + ) + model = TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=3e-3) + order = random.Random(0) # identical batch order across arms + record_by_id = {r.id: r for r in records} + model.train() + for _ in range(steps): + batch = [arm_rows[order.randrange(len(arm_rows))] for _ in range(batch_size)] + loss = state_supervision_loss(model, batch, record_by_id) + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +@torch.no_grad() +def valid_final_rate( + model: TreeEditDiffusionModel, held_out: list[ExampleRecord] +) -> dict: + """Decode from the seed per held-out record; fraction whose final beam + state is a valid program.""" + n_valid = 0 + finals: list[dict] = [] + for record in held_out: + inventory = [p if p.startswith(":") else f":{p}" for p in record.placeholders] + prompt = model._format_context( + record.prompt, design_md=record.design_md, slot_contract=inventory + ) + ctx, ctx_pad = model._encode_context([prompt]) + final_text, evidence = model._decode_one(ctx, ctx_pad, inventory) + ok = bool(final_text) and _is_valid(final_text) + n_valid += int(ok) + finals.append( + { + "record_id": record.id, + "valid": ok, + "steps": evidence.get("steps"), + "verifier_calls": evidence.get("verifier_calls"), + } + ) + return { + "n": len(held_out), + "valid_final_rate": n_valid / max(len(held_out), 1), + "finals": finals, + } + + +def build_report( + train_records: list[ExampleRecord], + held_out: list[ExampleRecord], + *, + steps: int, + batch_size: int, + per_arm_budget: int, + seed: int, + snapshot_out: Path, +) -> dict: + space = TreeEditSpace() + clear_caches() + # Tiny CPU fixture: torch thread thrash on tiny ops dominates wall-clock + # (measured 6x slower at 12 threads); pin to a single thread, identical + # for every arm, and declare it in the payload. + torch.set_num_threads(1) + start = time.perf_counter() + + # --- held-out eval states (labeled once, shared across arms) ---------- + eval_states = build_heldout_eval_states(held_out, space) + oracle = label_eval_states(eval_states, space) + held_fingerprints = { + source_fingerprint(render_statements(entry["statements"])) + for entry in eval_states + } + + # --- bootstrap model for on-policy rollouts (gold_only supervision) --- + bootstrap_rows = acquire_gold_only( + train_records, space, parent_checkpoint="none", suite_id=SUITE_ID + ) + bootstrap = train_arm( + train_records, + bootstrap_rows, + steps=steps, + batch_size=batch_size, + seed=seed, + ) + + # --- acquire all sources over TRAIN records only ---------------------- + rows = [] + rows += bootstrap_rows + rows += acquire_seedward( + train_records, space, parent_checkpoint="none", suite_id=SUITE_ID + ) + rows += acquire_on_policy( + bootstrap, + train_records, + space, + parent_checkpoint="bootstrap_gold_only_fixture", + suite_id=SUITE_ID, + max_states_per_record=ON_POLICY_MAX_STATES_PER_RECORD, + ) + + # Drop legitimate cross-record fingerprint collisions (counted in the + # snapshot manifest), then the fail-closed guard verifies the kept set. + rows, dropped_leaks = drop_leaked_rows(rows, held_fingerprints) + assert_no_leakage(rows, held_fingerprints) + + snapshot = StateSnapshotV1( + snapshot_id=f"{EXPERIMENT_ID}-fixture", + config={ + "suite_id": SUITE_ID, + "arms": PREREGISTERED_ARMS, + "per_arm_budget": per_arm_budget, + "steps": steps, + "batch_size": batch_size, + "seed": seed, + "held_out_fingerprints": sorted(held_fingerprints), + "dropped_leak_collisions": dropped_leaks, + }, + rows=rows, + ) + snapshot.write(snapshot_out) + coverage = source_coverage(rows) + + # --- matched arms ------------------------------------------------------- + arms: dict[str, dict] = {} + for arm in sorted(PREREGISTERED_ARMS): + arm_rows = build_arm_rows(rows, arm, per_arm_budget=per_arm_budget, seed=seed) + model = train_arm( + train_records, arm_rows, steps=steps, batch_size=batch_size, seed=seed + ) + metrics = score_arm(model, held_out, eval_states) + arms[arm] = { + "n_rows": len(arm_rows), + "coverage": source_coverage(arm_rows), + "metrics": metrics, + "valid_final": valid_final_rate(model, held_out), + "training_metrics": model.last_training_metrics, + } + + gold = arms["gold_only"]["metrics"] + best_name = None + best_regret_imp: float | None = None + for name in ("seedward", "on_policy", "mixed"): + a = arms[name]["metrics"] + if a["beam_regret_mean"] is None or gold["beam_regret_mean"] is None: + continue + imp = gold["beam_regret_mean"] - a["beam_regret_mean"] + if best_regret_imp is None or imp > best_regret_imp: + best_regret_imp = imp + best_name = name + rank_degradation = None + if ( + best_name is not None + and gold["rank_correlation"] is not None + and arms[best_name]["metrics"]["rank_correlation"] is not None + ): + rank_degradation = ( + gold["rank_correlation"] - arms[best_name]["metrics"]["rank_correlation"] + ) + regret_ok = ( + best_regret_imp is not None + and best_regret_imp >= PREREGISTERED_THRESHOLDS["beam_regret_improvement_min"] + ) + rank_ok = ( + rank_degradation is not None + and rank_degradation + <= PREREGISTERED_THRESHOLDS["rank_correlation_degradation_max"] + ) + verdict = "distribution_gap_closed" if (regret_ok and rank_ok) else "rejected" + + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-312", + "question": ( + "Does closing the gold-corruption vs seed-rollout distribution gap " + "(seedward + on-policy state sources) improve repair more than " + "extra offline near-gold training at equal budget?" + ), + "preregistered_thresholds": PREREGISTERED_THRESHOLDS, + "preregistered_arms": PREREGISTERED_ARMS, + "config": { + "steps": steps, + "batch_size": batch_size, + "per_arm_budget": per_arm_budget, + "seed": seed, + "n_train_records": len(train_records), + "n_held_out_records": len(held_out), + "suite_id": SUITE_ID, + "fixture_config_overrides": FIXTURE_CONFIG_OVERRIDES, + "on_policy_max_states_per_record": ON_POLICY_MAX_STATES_PER_RECORD, + "torch_num_threads": 1, + }, + "snapshot": { + "path": str(snapshot_out), + "manifest": snapshot.manifest(), + "coverage": coverage, + }, + "oracle_cost": oracle, + "arms": arms, + "comparison": { + "best_non_gold_arm": best_name, + "beam_regret_improvement": best_regret_imp, + "rank_correlation_degradation": rank_degradation, + }, + "threshold_checks": {"beam_regret_ok": regret_ok, "rank_ok": rank_ok}, + "verdict": verdict, + "wall_seconds": time.perf_counter() - start, + "honesty": ( + "Fixture-scale matched arms over immutable content-addressed state " + "snapshots; evaluation uses held-out seed trajectories only " + "(fail-closed leak guard ran before training); UNKNOWN/unbounded " + "oracle labels are excluded and counted, never coerced. A fixture " + "verdict is wiring/state-source evidence, not a production ship " + "claim." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm312_state_sources", + "harness.experiments.slm299_edit_reachability", + ) + return payload + + +def render_markdown(payload: dict) -> str: + thr = payload["preregistered_thresholds"] + + def fmt(x: object) -> str: + return "n/a" if x is None else (f"{x:.4f}" if isinstance(x, float) else str(x)) + + lines = [ + "# SLM-312 (LAR2-04): seedward + on-policy state sources vs near-gold", + "", + f"**Verdict: `{payload['verdict']}`** (fixture-scale matched arms; not a ship claim)", + "", + "## Preregistered (locked before results)", + "", + f"- arms/weights: `{json.dumps(payload['preregistered_arms'])}`", + f"- per-arm budget: {payload['config']['per_arm_budget']} rows; " + f"identical model/steps/optimizer/seeds/evaluator", + f"- beam regret improvement >= {thr['beam_regret_improvement_min']}", + f"- rank correlation degradation <= {thr['rank_correlation_degradation_max']}", + f"- rule: {thr['verdict_rule']}", + "", + "## Headline (held-out seed trajectories)", + "", + "| arm | rows | rank corr | concordance | Brier | beam regret | valid-final |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for name, arm in sorted(payload["arms"].items()): + m = arm["metrics"] + lines.append( + f"| {name} | {arm['n_rows']} | {fmt(m['rank_correlation'])} | " + f"{fmt(m['concordance'])} | {fmt(m['brier'])} | " + f"{fmt(m['beam_regret_mean'])} | " + f"{fmt(arm['valid_final']['valid_final_rate'])} |" + ) + cmp_ = payload["comparison"] + lines += [ + "", + f"Best non-gold arm: **{cmp_['best_non_gold_arm']}** — beam regret " + f"improvement {fmt(cmp_['beam_regret_improvement'])}, rank correlation " + f"degradation {fmt(cmp_['rank_correlation_degradation'])}.", + "", + "## State-space coverage by source (full snapshot)", + "", + "| source | rows | unique fingerprints |", + "| --- | --- | --- |", + ] + cov = payload["snapshot"]["coverage"] + for source, row in cov["per_source"].items(): + lines.append(f"| {source} | {row['rows']} | {row['unique_fingerprints']} |") + lines += [ + "", + f"Snapshot duplicate rate {cov['duplicate_rate']:.3f}; " + f"{cov['unique_fingerprints']}/{cov['rows']} unique states. " + f"Manifest `{payload['snapshot']['manifest']['manifest_sha256'][:16]}…` " + f"at `{payload['snapshot']['path']}`.", + "", + "## Coverage + oracle cost", + "", + ] + oracle = payload["oracle_cost"] + lines.append( + f"- UNKNOWN coverage {oracle['unknown_coverage']:.3f} " + f"({oracle['n_unknown']}/{oracle['n_states']}), measurable " + f"{oracle['n_measurable']}, oracle {oracle['ms_per_state']:.1f} ms/state" + ) + lines += [ + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=4) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--per-arm-budget", type=int, default=24) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + parser.add_argument("--snapshot-out", type=Path, default=DEFAULT_SNAPSHOT_OUT) + args = parser.parse_args(argv) + + train_records, held_out = build_records() + payload = build_report( + train_records, + held_out, + steps=args.steps, + batch_size=args.batch_size, + per_arm_budget=args.per_arm_budget, + seed=args.seed, + snapshot_out=args.snapshot_out, + ) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"verdict={payload['verdict']} " + f"best_non_gold={payload['comparison']['best_non_gold_arm']} " + f"regret_imp={payload['comparison']['beam_regret_improvement']} " + f"-> {args.json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm312_state_sources.py b/src/slm_training/harnesses/experiments/slm312_state_sources.py new file mode 100644 index 000000000..b69552582 --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm312_state_sources.py @@ -0,0 +1,1150 @@ +"""SLM-312 (LAR2-04): seedward and on-policy state sources for tree-edit repair. + +Builds immutable, content-addressed training-state snapshots from three +sources over the SLM-305 extended edit space, labels every state with the +SLM-308 bounded distance oracle and an SLM-305 legal supervised edit, and +mixes them into matched ``{gold_only, seedward, on_policy, mixed}`` arms: + +- ``gold_only`` — the existing near-gold corruption chain (``sample_mutation`` + forward noise; the inverse edit is the supervised repair target). +- ``seedward`` — OFFLINE valid intermediates from the minimal decode seed, + reached by greedy oracle-guided edits TOWARD each gold target (SLM-308 + distance strictly decreasing). These cover the seed side of the + distribution that gold corruption never visits. +- ``on_policy`` — immutable snapshots of actual seed-originated beam + trajectories (``_decode_one`` states/proposals telemetry): visited states, + final states, and verifier-failure-cone states, each oracle-labeled. + +Honesty invariants: + +1. Every row carries explicit provenance (parent checkpoint, source commit, + rollout config, prompt/suite hashes, gold-visibility policy). On-policy + ROLLOUTS are gold-blind; oracle LABELS read gold at training time only — + the policy string says exactly which. +2. Leak guard is fail-closed: any AST-fingerprint overlap between a train row + and a held-out evaluation state raises :class:`LeakageError`. +3. Snapshots are content-addressed (per-row sha256 + manifest sha256); a + tampered row or manifest fails :func:`read_snapshot` with + :class:`SnapshotIntegrityError`. +4. UNKNOWN / unbounded oracle labels are excluded from value supervision, + never coerced (same rule as SLM-308). + +This module is torch-free at import time; the loss helper imports torch +lazily so acquisition/guard tests stay cheap. +""" + +from __future__ import annotations + +import hashlib +import json +import random +import subprocess +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Iterable, Sequence + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm299_edit_reachability import ( + _canonical_key, +) +from slm_training.harnesses.experiments.slm308_distance_oracle import ( + DistanceKind, + DistanceLabel, + distance_to_target, + effective_distance, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_ADD, + ACTION_ADD_CONTAINER, + ACTION_BIND_PLACEHOLDER, + ACTION_INSERT_STATEMENT, + ACTION_INSERT_SUBTREE, + ACTION_NAMES, + ACTION_REMOVE, + ACTION_REMOVE_CONTAINER, + ACTION_REPLACE, + ACTION_REPLACE_STATEMENT, + ACTION_REPLACE_SUBTREE, + CONTAINER_RESTS, + Edit, + Statement, + TreeEditSpace, + V05_TEMPLATES, + MAX_SLOTS, + parse_statements, + render_statements, +) + +EXPERIMENT_ID = "slm312-state-sources" +SNAPSHOT_SCHEMA_VERSION = 1 + +SOURCE_GOLD_ONLY = "gold_only" +SOURCE_SEEDWARD = "seedward" +SOURCE_ON_POLICY = "on_policy" +SOURCES = (SOURCE_GOLD_ONLY, SOURCE_SEEDWARD, SOURCE_ON_POLICY) + +# Gold-visibility policies (per row; the on-policy ROLLOUT is gold-blind — +# decode never reads gold — while oracle labels read gold at training time +# only, so the policy string states both phases explicitly). +GOLD_VISIBILITY_ACQUISITION_AND_LABELS = "acquisition_and_labels" +GOLD_VISIBILITY_LABELS_ONLY = "rollout_gold_blind_labels_gold_visible" + +# Oracle budgets for state-source labels (same shallow budgets as SLM-308 +# training-time labels; UNKNOWN is excluded downstream, never coerced). +ORACLE_MAX_DEPTH = 8 +ORACLE_NODE_BUDGET = 8 + +# PREREGISTERED mixture arms (locked before any training; deviations are +# append-only and exploratory). Weights are per-source row-mixture weights. +PREREGISTERED_ARMS: dict[str, dict[str, float]] = { + "gold_only": {SOURCE_GOLD_ONLY: 1.0}, + "seedward": {SOURCE_SEEDWARD: 1.0}, + "on_policy": {SOURCE_ON_POLICY: 1.0}, + "mixed": {SOURCE_GOLD_ONLY: 0.34, SOURCE_SEEDWARD: 0.33, SOURCE_ON_POLICY: 0.33}, +} + +SEED_SOURCE = 'root = Stack([], "column")' + + +class LeakageError(RuntimeError): + """A train row overlaps a held-out evaluation state (fail-closed).""" + + +class SnapshotIntegrityError(RuntimeError): + """A content-addressed snapshot row or manifest failed verification.""" + + +# --------------------------------------------------------------------------- # +# Hashing helpers +# --------------------------------------------------------------------------- # + + +def _canonical(obj: Any) -> str: + return json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str) + + +def _digest(obj: Any) -> str: + return hashlib.sha256(_canonical(obj).encode("utf-8")).hexdigest() + + +def ast_fingerprint(statements: list[Statement]) -> str: + """Alpha-invariant AST fingerprint (SLM-299 canonical key), sha256.""" + return hashlib.sha256(_canonical_key(statements).encode("utf-8")).hexdigest() + + +def source_fingerprint(source_text: str) -> str: + """Fingerprint of a rendered program; raises when it does not parse.""" + statements = parse_statements(source_text) + if statements is None: + raise ValueError(f"statements do not parse: {source_text!r}") + return ast_fingerprint(statements) + + +def _git_commit() -> str: + try: + out = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + timeout=10, + ) + return out.stdout.strip() + except Exception: # noqa: BLE001 + return "unknown" + + +def _text_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16] + + +# --------------------------------------------------------------------------- # +# Provenance + row schema +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class StateProvenance: + """Where one training state came from, and who was allowed to see gold.""" + + parent_checkpoint: str + source_commit: str + rollout_config: dict[str, Any] + prompt_hash: str + suite_hash: str + gold_visibility_policy: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "StateProvenance": + return cls( + parent_checkpoint=str(data["parent_checkpoint"]), + source_commit=str(data["source_commit"]), + rollout_config=dict(data.get("rollout_config") or {}), + prompt_hash=str(data["prompt_hash"]), + suite_hash=str(data["suite_hash"]), + gold_visibility_policy=str(data["gold_visibility_policy"]), + ) + + +def edit_to_dict(edit: Edit) -> dict[str, Any]: + return { + "action": edit.action, + "action_name": ACTION_NAMES[edit.action], + "stmt": edit.stmt, + "comp": edit.comp, + "slot": edit.slot, + "target": edit.target, + "payload": edit.payload, + } + + +def edit_from_dict(data: dict[str, Any]) -> Edit: + return Edit( + int(data["action"]), + stmt=int(data.get("stmt", 0)), + comp=int(data.get("comp", 0)), + slot=int(data.get("slot", 0)), + target=int(data.get("target", 0)), + payload=int(data.get("payload", 0)), + ) + + +@dataclass(frozen=True) +class StateSourceRow: + """One labeled training state from one source, immutable + addressed.""" + + record_id: str + split: str # "train" | "held_out" (held_out rows are eval-only) + source: str # one of SOURCES + kind: str # near_gold | seedward_step | visited | final | verifier_failure_cone + statements_source: str + inventory: tuple[str, ...] + supervised_edit: dict[str, Any] | None # SLM-305 legal edit toward gold + child_source: str | None # apply(supervised_edit); strictly closer when set + distance_label: dict[str, Any] # DistanceLabel.to_dict() + value_target: float | None # None => exclude from value loss + provenance: StateProvenance + schema_version: int = SNAPSHOT_SCHEMA_VERSION + + def content(self) -> dict[str, Any]: + data = asdict(self) + data["inventory"] = list(self.inventory) + data["provenance"] = self.provenance.to_dict() + return data + + def sha256(self) -> str: + return _digest(self.content()) + + def fingerprint(self) -> str: + return source_fingerprint(self.statements_source) + + def to_dict(self) -> dict[str, Any]: + data = self.content() + data["sha256"] = self.sha256() + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "StateSourceRow": + row = cls( + record_id=str(data["record_id"]), + split=str(data["split"]), + source=str(data["source"]), + kind=str(data["kind"]), + statements_source=str(data["statements_source"]), + inventory=tuple(str(x) for x in data.get("inventory") or ()), + supervised_edit=( + dict(data["supervised_edit"]) + if data.get("supervised_edit") is not None + else None + ), + child_source=data.get("child_source"), + distance_label=dict(data["distance_label"]), + value_target=( + float(data["value_target"]) + if data.get("value_target") is not None + else None + ), + provenance=StateProvenance.from_dict(data["provenance"]), + schema_version=int(data.get("schema_version", SNAPSHOT_SCHEMA_VERSION)), + ) + expected = data.get("sha256") + if expected is not None and expected != row.sha256(): + raise SnapshotIntegrityError( + f"row {row.record_id}/{row.kind} sha256 mismatch: " + f"stored {expected}, computed {row.sha256()}" + ) + return row + + +def make_provenance( + record: ExampleRecord, + *, + parent_checkpoint: str, + rollout_config: dict[str, Any], + suite_id: str, + gold_visibility_policy: str, + source_commit: str | None = None, +) -> StateProvenance: + return StateProvenance( + parent_checkpoint=parent_checkpoint, + source_commit=source_commit if source_commit is not None else _git_commit(), + rollout_config=dict(rollout_config), + prompt_hash=_text_hash(record.prompt), + suite_hash=_text_hash(suite_id), + gold_visibility_policy=gold_visibility_policy, + ) + + +def _label_row( + *, + record: ExampleRecord, + split: str, + source: str, + kind: str, + statements: list[Statement], + target: list[Statement], + inventory: Sequence[str], + supervised_edit: Edit | None, + space: TreeEditSpace, + provenance: StateProvenance, + witness: int | None, + child_source: str | None = None, + verify_edit: bool = True, +) -> StateSourceRow: + """Oracle-label one state. + + With ``verify_edit`` (default) the supervised edit must be SLM-305 legal + on this state (fail-closed ``apply``); the resulting child is the + pairwise-progress target. Gold-corruption rows instead pass the actual + chain parent as ``child_source`` with ``verify_edit=False`` — the + historical inverse edit is a supervision target in index space and is + never re-applied (same semantics as ``training_loss``). + """ + if supervised_edit is not None and verify_edit: + child = space.apply(statements, supervised_edit, list(inventory)) + if child is None: + raise ValueError( + f"supervised edit {edit_to_dict(supervised_edit)} is not legal " + f"for {render_statements(statements)!r} (fail-closed apply)" + ) + child_source = render_statements(child) + label = distance_to_target( + statements, + target, + space=space, + inventory=inventory, + max_depth=ORACLE_MAX_DEPTH, + node_budget=ORACLE_NODE_BUDGET, + upper_bound_witness=witness, + ) + return StateSourceRow( + record_id=record.id, + split=split, + source=source, + kind=kind, + statements_source=render_statements(statements), + inventory=tuple(inventory), + supervised_edit=( + edit_to_dict(supervised_edit) if supervised_edit is not None else None + ), + child_source=child_source, + distance_label=label.to_dict(), + value_target=label.value_target(ORACLE_MAX_DEPTH), + provenance=provenance, + ) + + +# --------------------------------------------------------------------------- # +# Leak guard (fail-closed) +# --------------------------------------------------------------------------- # + + +def drop_leaked_rows( + rows: Iterable[StateSourceRow], + held_out_fingerprints: Iterable[str], +) -> tuple[list[StateSourceRow], list[dict[str, str]]]: + """Remove TRAIN rows whose AST fingerprint collides with a held-out eval + state, returning (kept, dropped-log). Legitimate cross-record collisions + (alpha-invariant canonical keys make e.g. a one-leaf near-gold state of a + train record and a seed-walk state of a held-out record identical) are + dropped and counted — never silently kept. :func:`assert_no_leakage` + remains the fail-closed verification on the kept set.""" + held = set(held_out_fingerprints) + kept: list[StateSourceRow] = [] + dropped: list[dict[str, str]] = [] + for row in rows: + if row.split == "train" and row.fingerprint() in held: + dropped.append( + { + "record_id": row.record_id, + "source": row.source, + "kind": row.kind, + "fingerprint": row.fingerprint(), + "reason": "held_out_fingerprint_collision", + } + ) + continue + kept.append(row) + return kept, dropped + + +def assert_no_leakage( + rows: Iterable[StateSourceRow], + held_out_fingerprints: Iterable[str], +) -> None: + """Raise LeakageError when any TRAIN row's AST fingerprint appears in the + held-out evaluation fingerprint set. Held-out rows are eval-only and are + never train supervision, so only ``split == "train"`` rows are checked.""" + held = set(held_out_fingerprints) + if not held: + return + overlaps: list[dict[str, str]] = [] + for row in rows: + if row.split != "train": + continue + fp = row.fingerprint() + if fp in held: + overlaps.append( + { + "record_id": row.record_id, + "source": row.source, + "kind": row.kind, + "fingerprint": fp, + } + ) + if overlaps: + raise LeakageError( + f"{len(overlaps)} train row(s) overlap held-out eval states: " + + json.dumps(overlaps[:5], sort_keys=True) + ) + + +# --------------------------------------------------------------------------- # +# Child-edit enumeration (mirrors slm299's extended mode but keeps the Edit) +# --------------------------------------------------------------------------- # + + +def enumerate_child_edits( + space: TreeEditSpace, + statements: list[Statement], + inventory: Sequence[str], + *, + visited: set[str] | None = None, +) -> list[tuple[list[Statement], Edit]]: + """All one-edit successors under the deployed extended action set, + applied through ``TreeEditSpace.apply`` (fail-closed validity), keeping + the :class:`Edit` so it can be used as a supervision target. Mirrors + ``slm299_edit_reachability._enumerate_children(mode="extended")``.""" + from slm_training.models.tree_edit_diffusion import ( + CONTAINER_COMPONENTS, + LEAF_COMPONENTS, + ) + + children: list[tuple[list[Statement], Edit]] = [] + n_slots = min(len(inventory), MAX_SLOTS) + leaf_comp_idxs = [i for i, c in enumerate(space.components) if c in LEAF_COMPONENTS] + container_comp_idxs = [ + i for i, c in enumerate(space.components) if c in CONTAINER_COMPONENTS + ] + pre = None + if visited is not None: + pre = lambda working: _canonical_key(working) not in visited # noqa: E731 + by_name = {s.name: s for s in statements} + + def _replace_subtree_ok(stmt: Statement) -> bool: + if not stmt.has_list or len(stmt.children) != 1: + return False + leaf = by_name.get(stmt.children[0]) + return leaf is not None and not leaf.has_list + + for stmt_idx in range(len(statements)): + stmt = statements[stmt_idx] + for comp_idx in range(len(space.components)): + edit = Edit(ACTION_REPLACE, stmt_idx, comp_idx) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + for slot_idx in range(n_slots): + edit = Edit(ACTION_ADD, stmt_idx, comp_idx, slot_idx) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + if stmt.has_list: + for comp_idx in container_comp_idxs: + for rest_idx in range(len(CONTAINER_RESTS)): + edit = Edit( + ACTION_ADD_CONTAINER, stmt_idx, comp_idx, target=rest_idx + ) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + for comp_idx in container_comp_idxs: + for slot_idx in range(n_slots): + for payload in leaf_comp_idxs: + for rest_idx in range(len(CONTAINER_RESTS)): + edit = Edit( + ACTION_INSERT_SUBTREE, + stmt_idx, + comp_idx, + slot_idx, + target=rest_idx, + payload=payload, + ) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + if _replace_subtree_ok(stmt): + for slot_idx in range(n_slots): + for payload in leaf_comp_idxs: + edit = Edit( + ACTION_REPLACE_SUBTREE, + stmt_idx, + slot=slot_idx, + payload=payload, + ) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + else: + for slot_idx in range(n_slots): + edit = Edit(ACTION_BIND_PLACEHOLDER, stmt_idx, slot=slot_idx) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + edit = Edit(ACTION_REMOVE_CONTAINER, stmt_idx) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + edit = Edit(ACTION_REMOVE, stmt_idx) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + for payload in range(len(V05_TEMPLATES)): + edit = Edit(ACTION_REPLACE_STATEMENT, stmt_idx, payload=payload) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + for payload in range(len(V05_TEMPLATES)): + edit = Edit(ACTION_INSERT_STATEMENT, payload=payload) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + return children + + +def oracle_best_child( + statements: list[Statement], + target: list[Statement], + space: TreeEditSpace, + inventory: Sequence[str], +) -> tuple[Edit, list[Statement], DistanceLabel] | None: + """Strictly-improving child with the smallest oracle distance, or None. + + Deterministic: children are enumerated in fixed order and ties keep the + first. Returns the edit (a legal SLM-305 action by construction), the + child statements, and the child's distance label. + """ + current = distance_to_target( + statements, + target, + space=space, + inventory=inventory, + max_depth=ORACLE_MAX_DEPTH, + node_budget=ORACLE_NODE_BUDGET, + ) + d_current = effective_distance(current) + best: tuple[float, Edit, list[Statement], DistanceLabel] | None = None + for child, edit in enumerate_child_edits(space, statements, inventory): + label = distance_to_target( + child, + target, + space=space, + inventory=inventory, + max_depth=ORACLE_MAX_DEPTH, + node_budget=ORACLE_NODE_BUDGET, + ) + d = effective_distance(label) + if d is None: + continue + if d_current is not None and d >= d_current: + continue + if best is None or d < best[0]: + best = (d, edit, child, label) + if best is None: + return None + return best[1], best[2], best[3] + + +# --------------------------------------------------------------------------- # +# Acquisition +# --------------------------------------------------------------------------- # + + +def _inventory_of(record: ExampleRecord) -> list[str]: + return [p if p.startswith(":") else f":{p}" for p in (record.placeholders or ())] + + +def acquire_gold_only( + records: Sequence[ExampleRecord], + space: TreeEditSpace, + *, + split: str = "train", + k_max: int = 3, + seed: int = 20260724, + parent_checkpoint: str = "none", + suite_id: str = "slm312-fixture", +) -> list[StateSourceRow]: + """Existing near-gold corruption chain: mutate gold 1..k edits; the + inverse edit is the supervised repair target (current training path).""" + rows: list[StateSourceRow] = [] + for idx, record in enumerate(records): + gold = parse_statements(record.openui or "") + if gold is None: + continue + inventory = _inventory_of(record) + provenance = make_provenance( + record, + parent_checkpoint=parent_checkpoint, + rollout_config={"acquisition": "sample_mutation", "k_max": k_max}, + suite_id=suite_id, + gold_visibility_policy=GOLD_VISIBILITY_ACQUISITION_AND_LABELS, + ) + rng = random.Random(seed + idx) + for k in (1, 2, 3): + if k > k_max: + break + current = gold + prev = None + inverse: Edit | None = None + applied = 0 + for _ in range(k): + step = space.sample_mutation(current, inventory, rng) + if step is None: + break + prev = current + current, inverse = step + applied += 1 + if inverse is None: + continue + rows.append( + _label_row( + record=record, + split=split, + source=SOURCE_GOLD_ONLY, + kind="near_gold", + statements=current, + target=gold, + inventory=inventory, + supervised_edit=inverse, + space=space, + provenance=provenance, + witness=applied, + # Pairwise child = the actual chain parent (one inverse + # step closer to gold), mirroring training_loss; the + # inverse edit itself is an index-space CE target and is + # never re-applied there, so it is not re-applied here. + child_source=render_statements(prev) if prev is not None else None, + verify_edit=False, + ) + ) + return rows + + +def acquire_seedward( + records: Sequence[ExampleRecord], + space: TreeEditSpace, + *, + split: str = "train", + seed_source: str = SEED_SOURCE, + max_edits: int = 4, + parent_checkpoint: str = "none", + suite_id: str = "slm312-fixture", +) -> list[StateSourceRow]: + """OFFLINE seedward source: from the minimal decode seed, walk greedy + oracle-guided edits TOWARD each gold target, emitting every valid + intermediate (the seed side of the distribution). Every step is a legal + SLM-305 action and strictly distance-decreasing by construction.""" + rows: list[StateSourceRow] = [] + seed_statements = parse_statements(seed_source) + if seed_statements is None: + raise ValueError(f"seed source does not parse: {seed_source!r}") + for record in records: + gold = parse_statements(record.openui or "") + if gold is None: + continue + inventory = _inventory_of(record) + provenance = make_provenance( + record, + parent_checkpoint=parent_checkpoint, + rollout_config={ + "acquisition": "oracle_guided_greedy", + "seed_source": seed_source, + "max_edits": max_edits, + }, + suite_id=suite_id, + gold_visibility_policy=GOLD_VISIBILITY_ACQUISITION_AND_LABELS, + ) + current = seed_statements + chain: list[tuple[list[Statement], Edit | None]] = [] # (state, edit to next) + for _ in range(max_edits): + found = oracle_best_child(current, gold, space, inventory) + if found is None: + break + edit, child, _label = found + chain.append((child, edit)) + current = child + # Row i's supervised edit is the edit taken NEXT (toward gold); the + # final chain state gets its own oracle-best continuation, or STOP. + for i, (state, _took) in enumerate(chain): + if i + 1 < len(chain): + supervised: Edit | None = chain[i + 1][1] + else: + nxt = oracle_best_child(state, gold, space, inventory) + supervised = nxt[0] if nxt is not None else None + rows.append( + _label_row( + record=record, + split=split, + source=SOURCE_SEEDWARD, + kind="seedward_step", + statements=state, + target=gold, + inventory=inventory, + supervised_edit=supervised, + space=space, + provenance=provenance, + # Greedy seed-side steps do not prove a path to gold, so + # there is no honest upper-bound witness here. + witness=None, + ) + ) + return rows + + +def acquire_on_policy( + model: Any, + records: Sequence[ExampleRecord], + space: TreeEditSpace, + *, + split: str = "train", + parent_checkpoint: str, + suite_id: str = "slm312-fixture", + max_states_per_record: int = 8, +) -> list[StateSourceRow]: + """Immutable snapshots from ACTUAL seed-originated beam trajectories. + + Runs ``model._decode_one`` per record (gold-blind rollout), then labels + the visited/final states with the training-time oracle. Verifier-failure + cone states (states with at least one dead proposal) are tagged so the + failure neighborhoods are represented and are prioritized under + ``max_states_per_record`` (oracle-guided relabeling is search-backed, so + the per-record cap keeps acquisition cost bounded at fixture scale). + Rows are deduped by fingerprint, keeping the richest kind tag. + """ + rows: list[StateSourceRow] = [] + seen: set[str] = set() + rollout_config = { + "beam_width": model.config.beam_width, + "expand_per_state": model.config.expand_per_state, + "max_search_steps": model.config.max_search_steps, + "stop_slot_accounting": model.config.stop_slot_accounting, + } + for record in records: + gold = parse_statements(record.openui or "") + if gold is None: + continue + inventory = _inventory_of(record) + provenance = make_provenance( + record, + parent_checkpoint=parent_checkpoint, + rollout_config=rollout_config, + suite_id=suite_id, + gold_visibility_policy=GOLD_VISIBILITY_LABELS_ONLY, + ) + prompt = model._format_context( + record.prompt, design_md=record.design_md, slot_contract=inventory + ) + ctx, ctx_pad = model._encode_context([prompt]) + final_text, evidence = model._decode_one(ctx, ctx_pad, inventory) + if not final_text: + continue + # States with at least one dead (verifier-rejected) proposal. + failure_states = { + (p["step"], p["beam_row"]) + for p in evidence.get("proposals", []) + if not p.get("applicable", False) + } + candidates: list[tuple[str, str]] = [] # (kind, source) + for entry in evidence.get("states", []): + kind = ( + "verifier_failure_cone" + if (entry["step"], entry["beam_row"]) in failure_states + else "visited" + ) + candidates.append((kind, entry["source"])) + candidates.append(("final", final_text)) + # Prioritize failure-cone + final states under the per-record cap. + priority = {"verifier_failure_cone": 0, "final": 1, "visited": 2} + candidates.sort(key=lambda item: priority.get(item[0], 3)) + per_record = 0 + for kind, source in candidates: + if per_record >= max_states_per_record: + break + statements = parse_statements(source) + if statements is None: + continue + fp = ast_fingerprint(statements) + if fp in seen: + continue + seen.add(fp) + per_record += 1 + found = oracle_best_child(statements, gold, space, inventory) + edit = found[0] if found is not None else None + rows.append( + _label_row( + record=record, + split=split, + source=SOURCE_ON_POLICY, + kind=kind, + statements=statements, + target=gold, + inventory=inventory, + supervised_edit=edit, + space=space, + provenance=provenance, + witness=None, + ) + ) + return rows + + +# --------------------------------------------------------------------------- # +# Immutable content-addressed snapshots +# --------------------------------------------------------------------------- # + + +@dataclass +class StateSnapshotV1: + """A frozen set of labeled state rows with a content-address manifest.""" + + snapshot_id: str + config: dict[str, Any] + rows: list[StateSourceRow] = field(default_factory=list) + + def manifest(self) -> dict[str, Any]: + row_hashes = [r.sha256() for r in self.rows] + return { + "snapshot_schema": f"slm312-state-snapshot/v{SNAPSHOT_SCHEMA_VERSION}", + "snapshot_id": self.snapshot_id, + "config": self.config, + "n_rows": len(self.rows), + "row_sha256": row_hashes, + "manifest_sha256": _digest( + { + "snapshot_id": self.snapshot_id, + "config": self.config, + "row_sha256": row_hashes, + } + ), + } + + def write(self, path: Path) -> Path: + """Write rows as JSONL plus a ``.manifest.json`` sidecar.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + for row in self.rows: + fh.write(json.dumps(row.to_dict(), sort_keys=True) + "\n") + manifest_path = path.with_suffix(path.suffix + ".manifest.json") + manifest_path.write_text( + json.dumps(self.manifest(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return path + + +def read_snapshot(path: Path) -> StateSnapshotV1: + """Load a snapshot, verifying every row hash and the manifest (tamper + detection is fail-closed: any mismatch raises SnapshotIntegrityError).""" + manifest_path = path.with_suffix(path.suffix + ".manifest.json") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + rows: list[StateSourceRow] = [] + with path.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line: + rows.append(StateSourceRow.from_dict(json.loads(line))) + if [r.sha256() for r in rows] != list(manifest.get("row_sha256") or []): + raise SnapshotIntegrityError( + f"row hash sequence mismatch against manifest {manifest_path}" + ) + snapshot = StateSnapshotV1( + snapshot_id=str(manifest["snapshot_id"]), + config=dict(manifest.get("config") or {}), + rows=rows, + ) + if snapshot.manifest()["manifest_sha256"] != manifest.get("manifest_sha256"): + raise SnapshotIntegrityError(f"manifest sha256 mismatch for {manifest_path}") + return snapshot + + +# --------------------------------------------------------------------------- # +# Mixture arms (predeclared weights + caps BEFORE any training) +# --------------------------------------------------------------------------- # + + +def build_arm_rows( + rows: Sequence[StateSourceRow], + arm: str, + *, + per_arm_budget: int, + seed: int = 0, +) -> list[StateSourceRow]: + """Deterministic per-source sampling under the predeclared arm weights. + + Every arm draws at most ``per_arm_budget`` rows in total; the per-source + quota is ``round(weight * budget)`` (at least 1 for a positive weight), + sampled without replacement with a per-arm seed so arms share budgets but + not rows. + """ + if arm not in PREREGISTERED_ARMS: + raise ValueError( + f"unknown arm {arm!r} (predeclared: {sorted(PREREGISTERED_ARMS)})" + ) + weights = PREREGISTERED_ARMS[arm] + train_rows = [r for r in rows if r.split == "train"] + rng = random.Random(seed) + selected: list[StateSourceRow] = [] + for source in sorted(weights): + weight = weights[source] + if weight <= 0: + continue + quota = max(1, round(weight * per_arm_budget)) + pool = [r for r in train_rows if r.source == source] + # Dedupe by fingerprint within the source pool. + seen: set[str] = set() + unique: list[StateSourceRow] = [] + for row in pool: + fp = row.fingerprint() + if fp in seen: + continue + seen.add(fp) + unique.append(row) + n = min(quota, len(unique)) + selected.extend(rng.sample(unique, n) if n < len(unique) else unique) + return selected[:per_arm_budget] + + +def source_coverage(rows: Sequence[StateSourceRow]) -> dict[str, Any]: + """Per-source state-space coverage + AST-fingerprint duplicate rate.""" + per_source: dict[str, set[str]] = {} + counts: dict[str, int] = {} + for row in rows: + per_source.setdefault(row.source, set()).add(row.fingerprint()) + counts[row.source] = counts.get(row.source, 0) + 1 + total = len(rows) + unique_total = len({r.fingerprint() for r in rows}) + return { + "rows": total, + "unique_fingerprints": unique_total, + "duplicate_rate": (1.0 - unique_total / total) if total else 0.0, + "per_source": { + source: { + "rows": counts.get(source, 0), + "unique_fingerprints": len(fps), + } + for source, fps in sorted(per_source.items()) + }, + } + + +# --------------------------------------------------------------------------- # +# Training loss over acquired rows (mirrors training_loss mechanics) +# --------------------------------------------------------------------------- # + + +def state_supervision_loss( + model: Any, + rows: Sequence[StateSourceRow], + record_by_id: dict[str, ExampleRecord], +) -> Any: + """Edit-prediction + bounded-distance value loss over acquired state rows. + + Same loss terms as ``TreeEditDiffusionModel.training_loss`` (policy CE on + the supervised edit, value MSE with UNKNOWN masked out, pairwise progress + ranking toward the supervised child) but the states come from the arm's + snapshot rows instead of an internally sampled gold-corruption chain — + so the four arms share identical code and differ ONLY in state source. + Rows without a supervised edit (e.g. a state already at/oracle-closest to + gold) are supervised as STOP when the oracle proves distance 0, else they + carry value supervision only. + """ + import torch + import torch.nn.functional as F + + from slm_training.models.tree_edit_diffusion import ( + ACTION_STOP, + MAX_STMTS, + pairwise_progress_loss, + ) + + prompts: list[str] = [] + states: list[str] = [] + targets: list[Edit] = [] + values: list[float] = [] + value_mask: list[bool] = [] + pair_rows: list[tuple[int, str]] = [] + skipped = 0 + n_unknown_excluded = 0 + for row in rows: + record = record_by_id.get(row.record_id) + if record is None: + skipped += 1 + continue + statements = parse_statements(row.statements_source) + if statements is None: + skipped += 1 + continue + inventory = list(row.inventory) + prompt = model._format_context( + record.prompt, design_md=record.design_md, slot_contract=inventory + ) + prompts.append(prompt) + states.append(row.statements_source) + if row.supervised_edit is not None: + targets.append(edit_from_dict(row.supervised_edit)) + else: + targets.append(Edit(ACTION_STOP)) + if row.value_target is None: + values.append(0.0) + value_mask.append(False) + n_unknown_excluded += 1 + else: + values.append(float(row.value_target)) + value_mask.append(True) + if row.child_source is not None: + # Pairwise progress: the supervised child is strictly closer by + # construction (gold inverse edit or oracle-certified improving + # edit); the oracle labels on the row prove comparability. + d_parent = effective_distance( + DistanceLabel( + kind=DistanceKind(row.distance_label["kind"]), + reason=str(row.distance_label["reason"]), + distance=row.distance_label.get("distance"), + lo=row.distance_label.get("lo"), + hi=row.distance_label.get("hi"), + ) + ) + if d_parent is not None and d_parent > 0: + pair_rows.append((len(states) - 1, row.child_source)) + if not states: + return torch.zeros((), device=model.device_name, requires_grad=True) + n_main = len(states) + pair_child_rows = list(range(n_main, n_main + len(pair_rows))) + states.extend(child for _, child in pair_rows) + ctx, ctx_pad = model._encode_context(prompts) + if pair_rows: + ctx_rows = [row for row, _ in pair_rows] + ctx = torch.cat([ctx, ctx[ctx_rows]], dim=0) + ctx_pad = torch.cat([ctx_pad, ctx_pad[ctx_rows]], dim=0) + out = model.policy(model._state_batch(states), model.tokenizer.pad_id, ctx, ctx_pad) + device = model.device_name + action_t = torch.tensor([e.action for e in targets], device=device) + loss = F.cross_entropy(out["action"][:n_main], action_t) + losses = {"action": float(loss.detach().cpu())} + stmt_rows = [i for i, e in enumerate(targets) if e.action != ACTION_STOP] + if stmt_rows: + idx = torch.tensor(stmt_rows, device=device) + stmt_t = torch.tensor( + [min(targets[i].stmt, MAX_STMTS - 1) for i in stmt_rows], device=device + ) + stmt_loss = F.cross_entropy(out["stmt"][idx], stmt_t) + loss = loss + stmt_loss + losses["stmt"] = float(stmt_loss.detach().cpu()) + comp_rows = [ + i + for i, e in enumerate(targets) + if e.action + in {ACTION_REPLACE, ACTION_ADD, ACTION_ADD_CONTAINER, ACTION_INSERT_SUBTREE} + ] + if comp_rows: + idx = torch.tensor(comp_rows, device=device) + comp_t = torch.tensor([targets[i].comp for i in comp_rows], device=device) + comp_loss = F.cross_entropy(out["comp"][idx], comp_t) + loss = loss + comp_loss + losses["comp"] = float(comp_loss.detach().cpu()) + slot_rows = [ + i + for i, e in enumerate(targets) + if e.action + in { + ACTION_ADD, + ACTION_INSERT_SUBTREE, + ACTION_REPLACE_SUBTREE, + ACTION_BIND_PLACEHOLDER, + } + ] + if slot_rows: + idx = torch.tensor(slot_rows, device=device) + slot_t = torch.tensor( + [min(targets[i].slot, MAX_SLOTS - 1) for i in slot_rows], device=device + ) + slot_loss = F.cross_entropy(out["slot"][idx], slot_t) + loss = loss + slot_loss + losses["slot"] = float(slot_loss.detach().cpu()) + value_t = torch.tensor(values, device=device, dtype=out["value"].dtype) + mask_t = torch.tensor(value_mask, device=device, dtype=torch.bool) + if bool(mask_t.any()): + value_loss = F.mse_loss(out["value"][:n_main][mask_t], value_t[mask_t]) + loss = loss + value_loss + losses["value"] = float(value_loss.detach().cpu()) + if pair_rows: + pair_loss = pairwise_progress_loss( + out["value"][torch.tensor([row for row, _ in pair_rows], device=device)], + out["value"][torch.tensor(pair_child_rows, device=device)], + margin=model.config.pairwise_progress_margin, + ) + loss = loss + pair_loss + losses["pairwise_progress"] = float(pair_loss.detach().cpu()) + losses["value_unknown_excluded"] = float(n_unknown_excluded) + losses["skipped"] = float(skipped) + losses["rows"] = float(n_main) + model.last_training_metrics = losses + return loss + + +__all__ = [ + "EXPERIMENT_ID", + "GOLD_VISIBILITY_ACQUISITION_AND_LABELS", + "GOLD_VISIBILITY_LABELS_ONLY", + "LeakageError", + "PREREGISTERED_ARMS", + "SEED_SOURCE", + "SOURCES", + "SOURCE_GOLD_ONLY", + "SOURCE_ON_POLICY", + "SOURCE_SEEDWARD", + "SNAPSHOT_SCHEMA_VERSION", + "SnapshotIntegrityError", + "StateProvenance", + "StateSnapshotV1", + "StateSourceRow", + "acquire_gold_only", + "acquire_on_policy", + "acquire_seedward", + "assert_no_leakage", + "ast_fingerprint", + "build_arm_rows", + "drop_leaked_rows", + "edit_from_dict", + "edit_to_dict", + "enumerate_child_edits", + "make_provenance", + "oracle_best_child", + "read_snapshot", + "source_coverage", + "source_fingerprint", + "state_supervision_loss", +] diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 015d3f848..1ec0d23fb 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -7841,6 +7841,24 @@ "note": "initial registration; SLM-310 LAR2-03 action alignment: reason-coded proposal telemetry, train-target vs decode-demand distribution audit, declared inverse-action corruption sampler, legacy/corrected STOP-slot accounting arms, matched 2x2 fixture experiment with preregistered thresholds" } ] + }, + "harness.experiments.slm312_state_sources": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm312_state_sources.py", + "scripts/run_slm312_state_sources.py", + "tests/test_harnesses/experiments/test_slm312_state_sources.py", + "docs/design/iter-slm312-state-sources-20260724.json", + "docs/design/iter-slm312-state-sources-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-312 LAR2-04 state sources: offline seedward oracle-guided valid intermediates + immutable content-addressed on-policy beam-trajectory snapshots labeled with the SLM-308 distance oracle and SLM-305 legal edits, fail-closed AST-fingerprint leak guards, matched {gold_only, seedward, on_policy, mixed} mixture arms with preregistered thresholds" + } + ] } } } diff --git a/tests/test_harnesses/experiments/test_slm312_state_sources.py b/tests/test_harnesses/experiments/test_slm312_state_sources.py new file mode 100644 index 000000000..9d61a1727 --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm312_state_sources.py @@ -0,0 +1,310 @@ +"""SLM-312 (LAR2-04): state-source acquisition, snapshots, leak guards, arms.""" + +from __future__ import annotations + +import json + +import pytest + +# Tiny CPU fixture models: torch thread thrash dominates otherwise. +import torch + +torch.set_num_threads(1) + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm308_distance_oracle import ( + DistanceKind, + clear_caches, + distance_to_target, + effective_distance, +) +from slm_training.harnesses.experiments.slm312_state_sources import ( + GOLD_VISIBILITY_ACQUISITION_AND_LABELS, + GOLD_VISIBILITY_LABELS_ONLY, + PREREGISTERED_ARMS, + SOURCE_SEEDWARD, + LeakageError, + SnapshotIntegrityError, + StateSnapshotV1, + acquire_gold_only, + acquire_seedward, + assert_no_leakage, + build_arm_rows, + drop_leaked_rows, + edit_from_dict, + oracle_best_child, + read_snapshot, + source_fingerprint, + state_supervision_loss, +) +from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + TreeEditSpace, + parse_statements, + render_statements, +) + +GOLD = 'root = Stack([t, b], "column")\nt = TextContent(":hero.title")\nb = Button(":cta.label")' +SEED = 'root = Stack([], "column")' + +# Tiny fixture model config (same overrides as the CLI) to keep tests fast. +_SMALL_CONFIG = { + "context_backend": "scratch", + "seed": 0, + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3, +} + + +@pytest.fixture() +def space(): + return TreeEditSpace() + + +@pytest.fixture(autouse=True) +def _clean_oracle_caches(): + clear_caches() + yield + clear_caches() + + +@pytest.fixture() +def records(): + return [ + ExampleRecord( + id="train_0", + prompt="hero with cta", + openui=GOLD, + placeholders=[":hero.title", ":cta.label"], + split="train", + ), + ExampleRecord( + id="train_1", + prompt="form with input", + openui='root = Form([i, s], "column")\ni = TextInput(":form.email")\ns = Button(":form.submit")', + placeholders=[":form.email", ":form.submit"], + split="train", + ), + ] + + +# --- provenance is explicit per row ------------------------------------------ + + +def test_provenance_explicit_per_row(space, records) -> None: + rows = acquire_gold_only(records, space, parent_checkpoint="ckpt-abc") + assert rows, "expected near-gold rows" + for row in rows: + prov = row.provenance + assert prov.parent_checkpoint == "ckpt-abc" + assert prov.source_commit + assert prov.prompt_hash + assert prov.suite_hash + assert prov.gold_visibility_policy == GOLD_VISIBILITY_ACQUISITION_AND_LABELS + assert prov.rollout_config["acquisition"] == "sample_mutation" + # Labels present: SLM-308 distance + SLM-305 legal supervised edit. + assert row.distance_label["kind"] in { + DistanceKind.EXACT.value, + DistanceKind.BOUNDED.value, + DistanceKind.UNKNOWN.value, + } + assert row.supervised_edit is not None + assert "action_name" in row.supervised_edit + + +def test_on_policy_provenance_is_gold_blind_rollout(space, records) -> None: + from slm_training.harnesses.experiments.slm312_state_sources import ( + acquire_on_policy, + ) + + torch_rows = acquire_gold_only(records[:1], space) + model = TreeEditDiffusionModel.from_records( + records[:1], + config=TreeEditDiffusionConfig(**_SMALL_CONFIG), + device="cpu", + ) + rows = acquire_on_policy( + model, + records[:1], + space, + parent_checkpoint="bootstrap-test", + max_states_per_record=3, + ) + assert rows, "expected on-policy rows from beam telemetry" + kinds = {r.kind for r in rows} + assert kinds <= {"visited", "final", "verifier_failure_cone"} + assert "final" in kinds + for row in rows: + assert row.provenance.parent_checkpoint == "bootstrap-test" + assert row.provenance.gold_visibility_policy == GOLD_VISIBILITY_LABELS_ONLY + assert row.provenance.rollout_config["beam_width"] == model.config.beam_width + assert torch_rows # sanity: scaffold reused + + +# --- leak guard: fail closed ------------------------------------------------- + + +def test_leak_guard_fails_closed_on_overlap(space, records) -> None: + rows = acquire_gold_only(records, space) + victim = rows[0] + with pytest.raises(LeakageError): + assert_no_leakage(rows, {victim.fingerprint()}) + # A disjoint held-out set passes. + assert_no_leakage(rows, {"0" * 64}) + # drop_leaked_rows removes the colliding row and the kept set verifies. + kept, dropped = drop_leaked_rows(rows, {victim.fingerprint()}) + assert len(kept) == len(rows) - 1 + assert dropped and dropped[0]["record_id"] == victim.record_id + assert_no_leakage(kept, {victim.fingerprint()}) + + +# --- snapshot immutability / content addressing ------------------------------- + + +def test_snapshot_roundtrip_and_tamper_detection(space, records, tmp_path) -> None: + rows = acquire_gold_only(records, space) + snap = StateSnapshotV1("snap-test", {"k": 1}, rows) + path = snap.write(tmp_path / "rows.jsonl") + back = read_snapshot(path) + assert back.rows == rows + assert back.manifest()["manifest_sha256"] == snap.manifest()["manifest_sha256"] + + # Tamper with one row payload (keep the stored hash): detect on load. + lines = path.read_text(encoding="utf-8").splitlines() + first = json.loads(lines[0]) + first["kind"] = "tampered" + lines[0] = json.dumps(first, sort_keys=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + with pytest.raises(SnapshotIntegrityError): + read_snapshot(path) + + # Tamper with the manifest only: also detected. + path2 = snap.write(tmp_path / "rows2.jsonl") + manifest_path = path2.with_suffix(path2.suffix + ".manifest.json") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["row_sha256"][0] = "0" * 64 + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(SnapshotIntegrityError): + read_snapshot(path2) + + +# --- seedward source: valid intermediates reaching toward the target --------- + + +def test_seedward_produces_valid_intermediates_toward_target(space, records) -> None: + rows = acquire_seedward(records[:1], space) + assert rows, "expected seedward intermediates" + gold = parse_statements(GOLD) + assert gold is not None + seed_fp = source_fingerprint(SEED) + distances: list[float] = [] + for row in rows: + assert row.source == SOURCE_SEEDWARD + statements = parse_statements(row.statements_source) + assert statements is not None + # Every intermediate is a valid, non-seed state strictly closer than + # the seed side start, oracle-verified. + assert row.fingerprint() != seed_fp + label = distance_to_target( + statements, gold, space=space, inventory=list(row.inventory) + ) + assert label.kind is DistanceKind.EXACT + eff = effective_distance(label) + assert eff is not None + distances.append(eff) + # Supervised edit is SLM-305 legal on this state (fail-closed apply) + # or the state is already at the target (STOP supervision). + if row.supervised_edit is not None: + child = space.apply( + statements, edit_from_dict(row.supervised_edit), list(row.inventory) + ) + assert child is not None + assert row.child_source == render_statements(child) + # The walk is strictly distance-decreasing toward the gold target. + assert distances == sorted(distances, reverse=True) + assert distances[0] > distances[-1] + + +def test_oracle_best_child_strictly_improves(space) -> None: + seed = parse_statements(SEED) + gold = parse_statements(GOLD) + assert seed is not None and gold is not None + inventory = [":hero.title", ":cta.label"] + found = oracle_best_child(seed, gold, space, inventory) + assert found is not None + edit, child, label = found + d_seed = effective_distance( + distance_to_target(seed, gold, space=space, inventory=inventory) + ) + d_child = effective_distance( + distance_to_target(child, gold, space=space, inventory=inventory) + ) + # The child is always oracle-measurable; when the parent is measurable + # too, the child is strictly closer (the seed is UNKNOWN at the shallow + # training budgets, so the comparison is conditional). + assert d_child is not None + if d_seed is not None: + assert d_child < d_seed + assert label.kind is DistanceKind.EXACT + # The returned edit is legal and reproduces the child exactly. + assert space.apply(seed, edit, inventory) is not None + assert render_statements(space.apply(seed, edit, inventory)) == render_statements( + child + ) + + +# --- mixture arms: shared budgets, isolation ---------------------------------- + + +def test_mixture_arms_share_budgets_and_sources(space, records) -> None: + rows = acquire_gold_only(records, space) + acquire_seedward(records, space) + budget = 6 + for arm, weights in PREREGISTERED_ARMS.items(): + selected = build_arm_rows(rows, arm, per_arm_budget=budget, seed=0) + assert len(selected) <= budget + sources = {r.source for r in selected} + expected = {s for s, w in weights.items() if w > 0} + assert sources <= expected + if arm != "mixed": + assert sources <= {arm} + # Determinism: same seed, same rows. + again = build_arm_rows(rows, "mixed", per_arm_budget=budget, seed=0) + first = build_arm_rows(rows, "mixed", per_arm_budget=budget, seed=0) + assert [r.sha256() for r in again] == [r.sha256() for r in first] + # Unknown arm is rejected (predeclared set only). + with pytest.raises(ValueError, match="unknown arm"): + build_arm_rows(rows, "sneaky_extra_arm", per_arm_budget=budget) + + +# --- determinism of acquisition ----------------------------------------------- + + +def test_acquisition_deterministic(space, records) -> None: + a = acquire_gold_only(records, space) + acquire_seedward(records, space) + clear_caches() + b = acquire_gold_only(records, space) + acquire_seedward(records, space) + assert [r.sha256() for r in a] == [r.sha256() for r in b] + + +# --- loss consumes rows (wiring) ---------------------------------------------- + + +def test_state_supervision_loss_runs_and_backprops(space, records) -> None: + rows = acquire_gold_only(records, space) + model = TreeEditDiffusionModel.from_records( + records, + config=TreeEditDiffusionConfig(**_SMALL_CONFIG), + device="cpu", + ) + record_by_id = {r.id: r for r in records} + loss = state_supervision_loss(model, rows[:4], record_by_id) + assert float(loss.detach()) >= 0.0 + loss.backward() + metrics = model.last_training_metrics + assert metrics["rows"] == 4.0 + assert "action" in metrics and "value" in metrics From f6b173105eb2f3e2e5fa2b5fea7e3af452cc2b83 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:51:01 -0500 Subject: [PATCH 08/10] SLM-314: preserve multiple hard-valid AST modes with winner-take-all training - Frozen multi-mode dataset (8 prompts x 2 verifier-accepted modes; canonical alpha-invariant AST fingerprints, sha256 rows + tamper-evident manifest; prompt count declared below ship-gate scale). - WTA/MCL loss: backpropagate only the min-loss mode per example + preregistered floor term (eps=0.1) on losing modes; durable per-example selection telemetry. No set-FTPO machinery. - Two-mode synthetic fixture proves single-gold CE collapses (p(mode B)=0.00004) while WTA retains both modes (p(B)=0.0083, p(A)=0.826 dominant); multi-gold spreads 50% mass to invalid hybrids. - Matched arms {single_gold, multi_gold, wta} identical budgets: coverage 0.750/0.500/0.500, hard-valid decode 1.0, v2 regression 0.000. Verdict rejected per preregistered coverage gate (WTA gain -0.25); at this scale single-gold lifts the sibling mode too. - 16 new tests; slm314 component v1. --- .../iter-slm314-winner-take-all-20260724.json | 844 ++++++++++++++++++ .../iter-slm314-winner-take-all-20260724.md | 36 + scripts/run_slm314_winner_take_all.py | 507 +++++++++++ .../experiments/slm314_winner_take_all.py | 547 ++++++++++++ .../data/slm314_multimode/modes.jsonl | 8 + .../modes.jsonl.manifest.json | 19 + src/slm_training/resources/versions.json | 20 + .../test_slm314_winner_take_all.py | 244 +++++ 8 files changed, 2225 insertions(+) create mode 100644 docs/design/iter-slm314-winner-take-all-20260724.json create mode 100644 docs/design/iter-slm314-winner-take-all-20260724.md create mode 100644 scripts/run_slm314_winner_take_all.py create mode 100644 src/slm_training/harnesses/experiments/slm314_winner_take_all.py create mode 100644 src/slm_training/resources/data/slm314_multimode/modes.jsonl create mode 100644 src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json create mode 100644 tests/test_harnesses/experiments/test_slm314_winner_take_all.py diff --git a/docs/design/iter-slm314-winner-take-all-20260724.json b/docs/design/iter-slm314-winner-take-all-20260724.json new file mode 100644 index 000000000..cee2da10c --- /dev/null +++ b/docs/design/iter-slm314-winner-take-all-20260724.json @@ -0,0 +1,844 @@ +{ + "experiment": "slm314-winner-take-all", + "issue": "SLM-314", + "question": "Does winner-take-all (MCL) training with a preregistered floor term retain more hard-valid canonical AST modes than single-gold CE without a semantic regression, at equal budget?", + "preregistered_thresholds": { + "floor_epsilon": 0.1, + "retained_improvement_min": 0.0, + "retention_gap_margin": 0.5, + "semantic_regression_max_points": 0.05, + "verdict_rule": "wta_preserves_modes iff coverage(wta) > coverage(single_gold) AND v2 semantic verdict regression vs single_gold <= 0.05, else rejected" + }, + "arms": [ + "single_gold", + "multi_gold", + "wta" + ], + "config": { + "steps": 5, + "lr": 0.003, + "seed": 0, + "n_prompts": 8, + "n_modes": 16, + "dataset": "src/slm_training/resources/data/slm314_multimode/modes.jsonl", + "dataset_manifest": { + "dataset_schema": "slm314-multimode-dataset/v1", + "n_prompts": 8, + "n_modes": 16, + "modes_per_prompt": [ + 2 + ], + "row_sha256": [ + "17bd182f4bddf979caabdf1248fe325418a4047575600ae6d10cad1c47314774", + "f6ac316055c35daefbdb864879ff504f269326d8d0bde64fc585b14d8be28882", + "015568fa00c3a12f895aeb58af0b85cb799e28f2c315d42fe02bfb573dbca89e", + "b22959ee91c136cbb9814a20ef9b4cf579924bab79c7c2f15fc3497e6ac9df62", + "35bd9d295405696d90b4748c671460a2db892552d824760b1a170ee56b436347", + "aad565646ad04affe14603d1eb0de57fed8d189e1ed2c7c360bf72ed8c78a5f6", + "708f07f6793b1832d3bee49b1f62fa2ab7ef3157b8de5f938c631888fa056b99", + "4b29cac7b71e9aff55a1914f674bc5f54656e01210bdcb6db714f34abccd73fd" + ], + "manifest_sha256": "10d1edc939cc74c159858bf423aa3b457d691526efc41a125c50c5e97c7d42f9" + }, + "fixture_config_overrides": { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 3, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3 + }, + "torch_num_threads": 1 + }, + "synthetic_two_mode": { + "probabilities": { + "single_gold": { + "p_mode_a": 0.9873679876327515, + "p_mode_b": 4.0145620005205274e-05, + "p_invalid_hybrid": 0.012591822072863579 + }, + "multi_gold": { + "p_mode_a": 0.25, + "p_mode_b": 0.25, + "p_invalid_hybrid": 0.5 + }, + "wta": { + "p_mode_a": 0.8264458775520325, + "p_mode_b": 0.008264509029686451, + "p_invalid_hybrid": 0.16528968513011932 + } + }, + "assertions": { + "single_gold_collapses": true, + "wta_retains_loser_mode": true, + "wta_winner_dominant": true, + "multi_gold_hybrid_invalid": true + }, + "all_assertions_pass": true + }, + "arm_results": { + "single_gold": { + "per_mode": { + "n_modes": 16, + "n_retained": 12, + "coverage": 0.75, + "collapse_indicator": 0.25, + "per_group": [ + { + "group_id": "mm_00", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "80c5eda1a6896342816a9794926f6f9595c3c2b2f4a36b5beb68cb7fd91b3341", + "eval_loss": 8.701364517211914, + "baseline_loss": 8.232380867004395, + "improvement": -0.46898365020751953, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "6c454c79cac92994ad15f07e1e43019515d4df7a4dab2b09ab88a7133a0ba913", + "eval_loss": 7.9676361083984375, + "baseline_loss": 9.193222045898438, + "improvement": 1.2255859375, + "retained": true + } + ] + }, + { + "group_id": "mm_01", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "8c94a6a1e407ac005e58301670afa9aaccd0c2b1a04552693289f51a5a48ddf3", + "eval_loss": 4.781002521514893, + "baseline_loss": 5.63767147064209, + "improvement": 0.8566689491271973, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "8b2366208036fe324c2584d98e62cd697d1e0c2e1d2fc4461f19788d6510d691", + "eval_loss": 1.9130592346191406, + "baseline_loss": 2.660956621170044, + "improvement": 0.7478973865509033, + "retained": true + } + ] + }, + { + "group_id": "mm_02", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "4d0cafaed2d1d426605275844c7ffe9cbccb8c017cf18b7c4bff0932199f6fc8", + "eval_loss": 4.843299865722656, + "baseline_loss": 5.75419807434082, + "improvement": 0.9108982086181641, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "29b4a54a4aa6b58ea3cd3ef25a8b282da7a08459de3959cd97443f1eec8e5d2d", + "eval_loss": 6.901802062988281, + "baseline_loss": 8.262958526611328, + "improvement": 1.3611564636230469, + "retained": true + } + ] + }, + { + "group_id": "mm_03", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "5a798e31a8ce98a2a1a616a7e61531c632036bfc150d151bd775edc8903bff5b", + "eval_loss": 11.471742630004883, + "baseline_loss": 12.366077423095703, + "improvement": 0.8943347930908203, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "97b12398d55fd0836abcb35011122985efeb8dfbaf47d9d8d211f331aa598952", + "eval_loss": 4.823297500610352, + "baseline_loss": 6.045340538024902, + "improvement": 1.2220430374145508, + "retained": true + } + ] + }, + { + "group_id": "mm_04", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "b8528408c1ecc2292846532b492e323dec85034f4d98c445a1bfcb22d5120f58", + "eval_loss": 4.774424076080322, + "baseline_loss": 5.668830394744873, + "improvement": 0.8944063186645508, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "9a2d5f9922a1612e18ac88aaed8a1c807f9d61f214ffab1787ede721e85db843", + "eval_loss": 5.237826347351074, + "baseline_loss": 6.058230876922607, + "improvement": 0.8204045295715332, + "retained": true + } + ] + }, + { + "group_id": "mm_05", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "13b7964bf2252bdbe82c5ce990ada9242baffb0f3b0b269be614ecefeedf6dd9", + "eval_loss": 6.782610893249512, + "baseline_loss": 12.469882011413574, + "improvement": 5.6872711181640625, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "1b9cd54f0cf1b9a5bac1f2c7bc8120ab78bccc0f7dc7c247705e065e7591c89a", + "eval_loss": 1.9315738677978516, + "baseline_loss": 2.6750247478485107, + "improvement": 0.7434508800506592, + "retained": false + } + ] + }, + { + "group_id": "mm_06", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "de0096e262a8a20e11bb2338b2b99f3b38fca2034d657bf90c39ee7287d7d958", + "eval_loss": 7.448744297027588, + "baseline_loss": 12.19206714630127, + "improvement": 4.743322849273682, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "54f7897efa57c43a2166b33229c3b1a8f043c020d84d78ef193c2aa4bfc0fd88", + "eval_loss": 4.377158164978027, + "baseline_loss": 6.348398208618164, + "improvement": 1.9712400436401367, + "retained": false + } + ] + }, + { + "group_id": "mm_07", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "d60cd88bf91091b9ffc0087cf970ba229baa342ba8b955b32c4a906da8e881a3", + "eval_loss": 4.873033046722412, + "baseline_loss": 6.304152488708496, + "improvement": 1.431119441986084, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "c7f2a0d70546000767905aa498e7eaea65cdbfe66095ad6e9d93eab629e01969", + "eval_loss": 8.81863784790039, + "baseline_loss": 12.366247177124023, + "improvement": 3.547609329223633, + "retained": true + } + ] + } + ] + }, + "decode": { + "n_prompts": 8, + "hard_valid_rate": 1.0, + "mode_hit_rate": 0.125, + "v2_verdict_rate": 0.0, + "per_group": [ + { + "group_id": "mm_00", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_01", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_02", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_03", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_04", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_05", + "valid": true, + "mode_hit": true, + "v2_verdict": false + }, + { + "group_id": "mm_06", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_07", + "valid": true, + "mode_hit": false, + "v2_verdict": false + } + ] + }, + "coverage": 0.75, + "collapse_indicator": 0.25, + "training_metrics": { + "action": 2.8104822635650635, + "stmt": 2.094724178314209, + "comp": 3.1520068645477295, + "slot": 0.7475554347038269, + "value": 0.013868428766727448, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + }, + "multi_gold": { + "per_mode": { + "n_modes": 16, + "n_retained": 8, + "coverage": 0.5, + "collapse_indicator": 0.5, + "per_group": [ + { + "group_id": "mm_00", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "80c5eda1a6896342816a9794926f6f9595c3c2b2f4a36b5beb68cb7fd91b3341", + "eval_loss": 7.716122627258301, + "baseline_loss": 8.232380867004395, + "improvement": 0.5162582397460938, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "6c454c79cac92994ad15f07e1e43019515d4df7a4dab2b09ab88a7133a0ba913", + "eval_loss": 6.424439907073975, + "baseline_loss": 9.193222045898438, + "improvement": 2.768782138824463, + "retained": true + } + ] + }, + { + "group_id": "mm_01", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "8c94a6a1e407ac005e58301670afa9aaccd0c2b1a04552693289f51a5a48ddf3", + "eval_loss": 3.271097183227539, + "baseline_loss": 5.63767147064209, + "improvement": 2.366574287414551, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "8b2366208036fe324c2584d98e62cd697d1e0c2e1d2fc4461f19788d6510d691", + "eval_loss": 2.3212692737579346, + "baseline_loss": 2.660956621170044, + "improvement": 0.3396873474121094, + "retained": false + } + ] + }, + { + "group_id": "mm_02", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "4d0cafaed2d1d426605275844c7ffe9cbccb8c017cf18b7c4bff0932199f6fc8", + "eval_loss": 3.285755157470703, + "baseline_loss": 5.75419807434082, + "improvement": 2.468442916870117, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "29b4a54a4aa6b58ea3cd3ef25a8b282da7a08459de3959cd97443f1eec8e5d2d", + "eval_loss": 7.695723056793213, + "baseline_loss": 8.262958526611328, + "improvement": 0.5672354698181152, + "retained": false + } + ] + }, + { + "group_id": "mm_03", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "5a798e31a8ce98a2a1a616a7e61531c632036bfc150d151bd775edc8903bff5b", + "eval_loss": 7.438523292541504, + "baseline_loss": 12.366077423095703, + "improvement": 4.927554130554199, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "97b12398d55fd0836abcb35011122985efeb8dfbaf47d9d8d211f331aa598952", + "eval_loss": 5.40690803527832, + "baseline_loss": 6.045340538024902, + "improvement": 0.638432502746582, + "retained": false + } + ] + }, + { + "group_id": "mm_04", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "b8528408c1ecc2292846532b492e323dec85034f4d98c445a1bfcb22d5120f58", + "eval_loss": 3.2084126472473145, + "baseline_loss": 5.668830394744873, + "improvement": 2.4604177474975586, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "9a2d5f9922a1612e18ac88aaed8a1c807f9d61f214ffab1787ede721e85db843", + "eval_loss": 7.189324855804443, + "baseline_loss": 6.058230876922607, + "improvement": -1.131093978881836, + "retained": false + } + ] + }, + { + "group_id": "mm_05", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "13b7964bf2252bdbe82c5ce990ada9242baffb0f3b0b269be614ecefeedf6dd9", + "eval_loss": 9.23392105102539, + "baseline_loss": 12.469882011413574, + "improvement": 3.2359609603881836, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "1b9cd54f0cf1b9a5bac1f2c7bc8120ab78bccc0f7dc7c247705e065e7591c89a", + "eval_loss": 2.3135123252868652, + "baseline_loss": 2.6750247478485107, + "improvement": 0.3615124225616455, + "retained": false + } + ] + }, + { + "group_id": "mm_06", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "de0096e262a8a20e11bb2338b2b99f3b38fca2034d657bf90c39ee7287d7d958", + "eval_loss": 8.623126029968262, + "baseline_loss": 12.19206714630127, + "improvement": 3.568941116333008, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "54f7897efa57c43a2166b33229c3b1a8f043c020d84d78ef193c2aa4bfc0fd88", + "eval_loss": 5.369912624359131, + "baseline_loss": 6.348398208618164, + "improvement": 0.9784855842590332, + "retained": false + } + ] + }, + { + "group_id": "mm_07", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "d60cd88bf91091b9ffc0087cf970ba229baa342ba8b955b32c4a906da8e881a3", + "eval_loss": 5.448168754577637, + "baseline_loss": 6.304152488708496, + "improvement": 0.8559837341308594, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "c7f2a0d70546000767905aa498e7eaea65cdbfe66095ad6e9d93eab629e01969", + "eval_loss": 6.5001139640808105, + "baseline_loss": 12.366247177124023, + "improvement": 5.866133213043213, + "retained": true + } + ] + } + ] + }, + "decode": { + "n_prompts": 8, + "hard_valid_rate": 1.0, + "mode_hit_rate": 0.0, + "v2_verdict_rate": 0.0, + "per_group": [ + { + "group_id": "mm_00", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_01", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_02", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_03", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_04", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_05", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_06", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_07", + "valid": true, + "mode_hit": false, + "v2_verdict": false + } + ] + }, + "coverage": 0.5, + "collapse_indicator": 0.5, + "training_metrics": { + "action": 2.5397448539733887, + "stmt": 1.467846393585205, + "comp": 1.6208996772766113, + "slot": 0.8510135412216187, + "value": 0.02060924842953682, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + }, + "wta": { + "per_mode": { + "n_modes": 16, + "n_retained": 8, + "coverage": 0.5, + "collapse_indicator": 0.5, + "per_group": [ + { + "group_id": "mm_00", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "80c5eda1a6896342816a9794926f6f9595c3c2b2f4a36b5beb68cb7fd91b3341", + "eval_loss": 7.4052324295043945, + "baseline_loss": 8.232380867004395, + "improvement": 0.8271484375, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "6c454c79cac92994ad15f07e1e43019515d4df7a4dab2b09ab88a7133a0ba913", + "eval_loss": 7.391336441040039, + "baseline_loss": 9.193222045898438, + "improvement": 1.8018856048583984, + "retained": true + } + ] + }, + { + "group_id": "mm_01", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "8c94a6a1e407ac005e58301670afa9aaccd0c2b1a04552693289f51a5a48ddf3", + "eval_loss": 2.9950571060180664, + "baseline_loss": 5.63767147064209, + "improvement": 2.6426143646240234, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "8b2366208036fe324c2584d98e62cd697d1e0c2e1d2fc4461f19788d6510d691", + "eval_loss": 2.2241077423095703, + "baseline_loss": 2.660956621170044, + "improvement": 0.43684887886047363, + "retained": false + } + ] + }, + { + "group_id": "mm_02", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "4d0cafaed2d1d426605275844c7ffe9cbccb8c017cf18b7c4bff0932199f6fc8", + "eval_loss": 3.1677842140197754, + "baseline_loss": 5.75419807434082, + "improvement": 2.586413860321045, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "29b4a54a4aa6b58ea3cd3ef25a8b282da7a08459de3959cd97443f1eec8e5d2d", + "eval_loss": 7.574008941650391, + "baseline_loss": 8.262958526611328, + "improvement": 0.6889495849609375, + "retained": false + } + ] + }, + { + "group_id": "mm_03", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "5a798e31a8ce98a2a1a616a7e61531c632036bfc150d151bd775edc8903bff5b", + "eval_loss": 8.58497428894043, + "baseline_loss": 12.366077423095703, + "improvement": 3.7811031341552734, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "97b12398d55fd0836abcb35011122985efeb8dfbaf47d9d8d211f331aa598952", + "eval_loss": 4.4556169509887695, + "baseline_loss": 6.045340538024902, + "improvement": 1.5897235870361328, + "retained": false + } + ] + }, + { + "group_id": "mm_04", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "b8528408c1ecc2292846532b492e323dec85034f4d98c445a1bfcb22d5120f58", + "eval_loss": 2.9114532470703125, + "baseline_loss": 5.668830394744873, + "improvement": 2.7573771476745605, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "9a2d5f9922a1612e18ac88aaed8a1c807f9d61f214ffab1787ede721e85db843", + "eval_loss": 6.242021560668945, + "baseline_loss": 6.058230876922607, + "improvement": -0.1837906837463379, + "retained": false + } + ] + }, + { + "group_id": "mm_05", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "13b7964bf2252bdbe82c5ce990ada9242baffb0f3b0b269be614ecefeedf6dd9", + "eval_loss": 9.684586524963379, + "baseline_loss": 12.469882011413574, + "improvement": 2.7852954864501953, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "1b9cd54f0cf1b9a5bac1f2c7bc8120ab78bccc0f7dc7c247705e065e7591c89a", + "eval_loss": 2.2146434783935547, + "baseline_loss": 2.6750247478485107, + "improvement": 0.46038126945495605, + "retained": false + } + ] + }, + { + "group_id": "mm_06", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "de0096e262a8a20e11bb2338b2b99f3b38fca2034d657bf90c39ee7287d7d958", + "eval_loss": 11.33463191986084, + "baseline_loss": 12.19206714630127, + "improvement": 0.8574352264404297, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "54f7897efa57c43a2166b33229c3b1a8f043c020d84d78ef193c2aa4bfc0fd88", + "eval_loss": 6.33472204208374, + "baseline_loss": 6.348398208618164, + "improvement": 0.013676166534423828, + "retained": false + } + ] + }, + { + "group_id": "mm_07", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "d60cd88bf91091b9ffc0087cf970ba229baa342ba8b955b32c4a906da8e881a3", + "eval_loss": 4.460836410522461, + "baseline_loss": 6.304152488708496, + "improvement": 1.8433160781860352, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "c7f2a0d70546000767905aa498e7eaea65cdbfe66095ad6e9d93eab629e01969", + "eval_loss": 8.877434730529785, + "baseline_loss": 12.366247177124023, + "improvement": 3.4888124465942383, + "retained": true + } + ] + } + ] + }, + "decode": { + "n_prompts": 8, + "hard_valid_rate": 1.0, + "mode_hit_rate": 0.125, + "v2_verdict_rate": 0.0, + "per_group": [ + { + "group_id": "mm_00", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_01", + "valid": true, + "mode_hit": true, + "v2_verdict": false + }, + { + "group_id": "mm_02", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_03", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_04", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_05", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_06", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_07", + "valid": true, + "mode_hit": false, + "v2_verdict": false + } + ] + }, + "coverage": 0.5, + "collapse_indicator": 0.5, + "training_metrics": { + "action": 3.007995367050171, + "stmt": 1.427188754081726, + "comp": 2.3815364837646484, + "slot": 2.056959390640259, + "value": 0.003755042562261224, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + }, + "comparison": { + "coverage_gain_wta_vs_single_gold": -0.25, + "semantic_regression_wta_vs_single_gold": 0.0, + "coverage_multi_gold": 0.5 + }, + "threshold_checks": { + "coverage_ok": false, + "semantic_ok": true, + "synthetic_ok": true + }, + "telemetry_path": "outputs/slm314/wta_telemetry.jsonl", + "verdict": "rejected", + "wall_seconds": 75.39187614799448, + "honesty": "Fixture-scale: 8 prompts x 2 modes is far below any ship-gate prompt count \u2014 mechanism evidence only, not a production ship claim. Mode identity is alpha-invariant canonical AST fingerprints (never serialization strings); the frozen dataset is hash-verified before training. All arms share model init, steps, optimizer, per-step prompt coverage, and seeds. Decode is deterministic beam search (unique valid ASTs per prompt == 1 by construction); coverage is measured by deterministic per-mode eval loss instead. The SLM-130 ambiguity sets are wiring reports without a committed multi-mode corpus, so this dataset is synthesized and frozen in-repo.", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "0dd815193a4fccb41f229751393ed18a989718d7", + "code_dirty": true, + "components": { + "harness.experiments.slm314_winner_take_all": "v1", + "harness.experiments.slm299_edit_reachability": "v4" + }, + "stamped_at": "2026-07-24T23:47:05.065449+00:00" + } +} diff --git a/docs/design/iter-slm314-winner-take-all-20260724.md b/docs/design/iter-slm314-winner-take-all-20260724.md new file mode 100644 index 000000000..3776ed4d4 --- /dev/null +++ b/docs/design/iter-slm314-winner-take-all-20260724.md @@ -0,0 +1,36 @@ +# SLM-314 (LAR2-05): winner-take-all over multi-mode gold + +**Verdict: `rejected`** (fixture-scale matched arms; not a ship claim) + +## Preregistered (locked before results) + +- arms: `["single_gold", "multi_gold", "wta"]` — identical model init, steps, optimizer, per-step prompt coverage, seeds +- floor epsilon: 0.1; retained iff eval-loss improvement > 0.0 vs matched baseline and within 0.5 of the prompt's best-improved mode +- rule: wta_preserves_modes iff coverage(wta) > coverage(single_gold) AND v2 semantic verdict regression vs single_gold <= 0.05, else rejected +- frozen dataset: `src/slm_training/resources/data/slm314_multimode/modes.jsonl` (8 prompts x 2 modes, manifest `10d1edc939cc74c1…`) + +## Synthetic two-mode proof (deterministic) + +| arm | p(mode A) | p(mode B) | p(invalid hybrid) | +| --- | --- | --- | --- | +| single_gold | 0.987 | 0.000 | 0.013 | +| multi_gold | 0.250 | 0.250 | 0.500 | +| wta | 0.826 | 0.008 | 0.165 | + +Assertions: single_gold_collapses=PASS, wta_retains_loser_mode=PASS, wta_winner_dominant=PASS, multi_gold_hybrid_invalid=PASS + +## Matched arms (real tree-edit model, frozen multi-mode dataset) + +| arm | mode coverage | collapse | hard-valid | mode-hit | v2 verdict | +| --- | --- | --- | --- | --- | --- | +| single_gold | 0.750 | 0.250 | 1.000 | 0.125 | 0.000 | +| multi_gold | 0.500 | 0.500 | 1.000 | 0.000 | 0.000 | +| wta | 0.500 | 0.500 | 1.000 | 0.125 | 0.000 | + +Coverage gain (wta − single_gold): **-0.250**; v2 semantic regression: **0.000** (budget 0.05). Multi-gold coverage: 0.500. + +WTA per-step selected-mode telemetry: `outputs/slm314/wta_telemetry.jsonl`. + +## Honesty + +Fixture-scale: 8 prompts x 2 modes is far below any ship-gate prompt count — mechanism evidence only, not a production ship claim. Mode identity is alpha-invariant canonical AST fingerprints (never serialization strings); the frozen dataset is hash-verified before training. All arms share model init, steps, optimizer, per-step prompt coverage, and seeds. Decode is deterministic beam search (unique valid ASTs per prompt == 1 by construction); coverage is measured by deterministic per-mode eval loss instead. The SLM-130 ambiguity sets are wiring reports without a committed multi-mode corpus, so this dataset is synthesized and frozen in-repo. diff --git a/scripts/run_slm314_winner_take_all.py b/scripts/run_slm314_winner_take_all.py new file mode 100644 index 000000000..304fcd933 --- /dev/null +++ b/scripts/run_slm314_winner_take_all.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""SLM-314 (LAR2-05): winner-take-all vs single-gold vs multi-gold CE. + +Question: when a prompt has multiple hard-valid canonical AST modes, does +winner-take-all (MCL) training — min-loss mode per example plus a small +preregistered coverage/floor term — retain MORE verifier-accepted modes than +single-gold CE (which drops all but one mode at the dataset level) without a +semantic regression, at equal model / steps / optimizer / seed budget? The +duplicated-example multi-gold CE arm is the averaging baseline. + +Preregistered (locked before any run; written into the payload): + +- frozen dataset: ``src/slm_training/resources/data/slm314_multimode/modes.jsonl`` + (8 prompts x 2 verifier-accepted canonical AST modes; mode identity = + alpha-invariant canonical AST fingerprints + sha256 freeze manifest); +- arms {single_gold, multi_gold, wta} with identical model init, steps, + optimizer, per-step prompt coverage, and seeds; +- WTA floor epsilon 0.1; a frozen mode counts as RETAINED when its + deterministic eval loss IMPROVES vs the matched untrained baseline (same + init seed, same fixed eval rng) by more than 0.0 AND keeps pace with its + prompt's best-improved mode within 0.5 loss units; +- primary verdict ``wta_preserves_modes`` iff coverage(wta) > + coverage(single_gold) AND v2 semantic verdict regression vs single_gold + <= 5 points; else ``rejected`` — honestly computed, never narrated; +- synthetic two-mode fixture assertions (collapse/retain/hybrid thresholds in + ``slm314_winner_take_all``) must hold as the mechanism proof. + +Evaluation per arm: per-mode retention (deterministic eval loss), mode +coverage, collapse indicator, decode hard-valid rate, decoded-AST mode-hit +rate (canonical fingerprint match against the prompt's frozen modes), and v2 +semantic verdict rate (``binding_aware_meaningful_v2``). Fixture-scale: 8 +prompts is far below any ship-gate prompt count — this is mechanism evidence, +not a ship claim. + +Writes ``docs/design/iter-slm314-winner-take-all-20260724.{json,md}`` and +durable WTA telemetry under ``outputs/slm314/``. + +Example: + python -m scripts.run_slm314_winner_take_all --steps 10 +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import torch + +from slm_training.dsl.schema import ExampleRecord +from slm_training.evals.meaningful_program import binding_aware_meaningful_v2 +from slm_training.harnesses.experiments.slm314_winner_take_all import ( + DEFAULT_RESOURCE, + EXPERIMENT_ID, + FLOOR_EPSILON, + PromptGroupV1, + _seed_int, + dataset_manifest, + mode_supervision_loss, + read_frozen_dataset, + run_synthetic_two_mode, + single_gold_groups, + synthetic_assertions, + winner_take_all_loss, +) +from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + _is_valid, + parse_statements, +) +from slm_training.harnesses.experiments.slm312_state_sources import ast_fingerprint +from slm_training.versioning import build_version_stamp + +DEFAULT_JSON_OUT = Path("docs/design/iter-slm314-winner-take-all-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm314-winner-take-all-20260724.md") +DEFAULT_TELEMETRY_OUT = Path("outputs/slm314/wta_telemetry.jsonl") + +ARMS = ("single_gold", "multi_gold", "wta") + +# Tiny fixture model/decode config — identical across ALL arms and declared in +# the payload; it only bounds fixture wall-clock (the run cap), never differs +# by arm. +FIXTURE_CONFIG_OVERRIDES = { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 3, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3, +} + +# PREREGISTERED thresholds (locked before any run; deviations are append-only +# and exploratory). +PREREGISTERED_THRESHOLDS = { + "floor_epsilon": FLOOR_EPSILON, + "retained_improvement_min": 0.0, + "retention_gap_margin": 0.5, + "semantic_regression_max_points": 0.05, + "verdict_rule": ( + "wta_preserves_modes iff coverage(wta) > coverage(single_gold) AND " + "v2 semantic verdict regression vs single_gold <= 0.05, else rejected" + ), +} + + +def build_model(records: list[ExampleRecord], *, seed: int) -> TreeEditDiffusionModel: + """Identical model init for every arm (same tokenizer texts, same seed).""" + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + seed=seed, + # mutation_count value labels: the SLM-308 bounded-distance oracle is + # too expensive for this fixture's per-mode WTA loss (measured 5x + # wall-clock); arms stay matched and the mode is declared. + value_label_mode="mutation_count", + context_backend="scratch", + **FIXTURE_CONFIG_OVERRIDES, + ) + return TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + + +def train_arm( + arm: str, + groups: list[PromptGroupV1], + records: list[ExampleRecord], + *, + steps: int, + lr: float, + seed: int, + telemetry_rows: list[dict] | None = None, +) -> TreeEditDiffusionModel: + """One arm. Every step covers every prompt group exactly once for all + arms (identical prompt budget); arms differ ONLY in loss weighting: + single_gold = one mode per prompt, multi_gold = plain CE over duplicated + per-mode examples, wta = min-loss mode + floor term.""" + model = build_model(records, seed=seed) + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=lr) + sg_records = [r for g in single_gold_groups(groups) for r in g.records()] + mg_records = [r for g in groups for r in g.records()] + model.train() + for step in range(steps): + if arm == "single_gold": + loss = model.training_loss(sg_records) + elif arm == "multi_gold": + loss = model.training_loss(mg_records) + else: + total = None + for group in groups: + group_loss, telemetry = winner_take_all_loss( + model, group, floor_epsilon=FLOOR_EPSILON, seed=seed + step + ) + total = group_loss if total is None else total + group_loss + if telemetry_rows is not None: + telemetry_rows.append({"step": step, **telemetry}) + loss = total + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +@torch.no_grad() +def per_mode_eval_losses( + model: TreeEditDiffusionModel, groups: list[PromptGroupV1] +) -> dict[tuple[str, str], float]: + """Deterministic per-mode eval loss (fixed eval rng, shared across arms).""" + losses: dict[tuple[str, str], float] = {} + for group in groups: + for mode, record in zip(group.modes, group.records()): + losses[(group.group_id, mode.canonical_fingerprint)] = float( + mode_supervision_loss( + model, + record, + seed=_seed_int( + "slm314-eval", group.group_id, mode.canonical_fingerprint + ), + ).detach().cpu() + ) + return losses + + +def eval_per_mode( + model: TreeEditDiffusionModel, + groups: list[PromptGroupV1], + baseline: dict[tuple[str, str], float], +) -> dict: + """A mode is RETAINED when its deterministic eval loss improves on the + matched untrained baseline (same init seed, same eval rng) by more than + the preregistered margin AND it keeps pace with its prompt's + best-improved mode within ``retention_gap_margin`` loss units.""" + margin = PREREGISTERED_THRESHOLDS["retained_improvement_min"] + gap = PREREGISTERED_THRESHOLDS["retention_gap_margin"] + losses = per_mode_eval_losses(model, groups) + per_group: list[dict] = [] + retained = 0 + total = 0 + for group in groups: + improvements = [ + baseline[(group.group_id, m.canonical_fingerprint)] + - losses[(group.group_id, m.canonical_fingerprint)] + for m in group.modes + ] + best = max(improvements) + rows = [] + for mode, improvement in zip(group.modes, improvements): + key = (group.group_id, mode.canonical_fingerprint) + loss = losses[key] + kept = improvement > margin and improvement >= best - gap + retained += int(kept) + total += 1 + rows.append( + { + "mode_index": mode.mode_index, + "canonical_fingerprint": mode.canonical_fingerprint, + "eval_loss": loss, + "baseline_loss": baseline[key], + "improvement": improvement, + "retained": kept, + } + ) + per_group.append({"group_id": group.group_id, "modes": rows}) + coverage = retained / max(total, 1) + return { + "n_modes": total, + "n_retained": retained, + "coverage": coverage, + "collapse_indicator": 1.0 - coverage, + "per_group": per_group, + } + + +@torch.no_grad() +def eval_decode( + model: TreeEditDiffusionModel, groups: list[PromptGroupV1] +) -> dict: + """Decode once per prompt (beam decode is deterministic): hard-valid rate, + mode-hit rate (decoded AST fingerprint in the prompt's frozen mode set), + and v2 semantic verdict rate.""" + n_valid = 0 + n_hit = 0 + n_v2 = 0 + rows = [] + for group in groups: + inventory = [p if p.startswith(":") else f":{p}" for p in group.modes[0].placeholders] + prompt = model._format_context(group.prompt, slot_contract=inventory) + ctx, ctx_pad = model._encode_context([prompt]) + text, _evidence = model._decode_one(ctx, ctx_pad, inventory) + valid = bool(text) and _is_valid(text) + hit = False + if valid: + statements = parse_statements(text) + if statements is not None: + fp = ast_fingerprint(statements) + hit = fp in {m.canonical_fingerprint for m in group.modes} + report = binding_aware_meaningful_v2( + text, + record=ExampleRecord( + id=group.group_id, + prompt=group.prompt, + openui=group.modes[0].source, + placeholders=list(inventory), + ), + ) + n_valid += int(valid) + n_hit += int(hit) + n_v2 += int(report.verdict) + rows.append( + { + "group_id": group.group_id, + "valid": valid, + "mode_hit": hit, + "v2_verdict": bool(report.verdict), + } + ) + n = max(len(groups), 1) + return { + "n_prompts": len(groups), + "hard_valid_rate": n_valid / n, + "mode_hit_rate": n_hit / n, + "v2_verdict_rate": n_v2 / n, + "per_group": rows, + } + + +def build_report( + groups: list[PromptGroupV1], + *, + steps: int, + lr: float, + seed: int, + telemetry_out: Path, +) -> dict: + torch.set_num_threads(1) # tiny CPU fixture; identical for every arm + start = time.perf_counter() + records = [r for g in groups for r in g.records()] + + # Matched untrained baseline (identical init seed/config): retention is + # measured as per-mode eval-loss IMPROVEMENT against this baseline. + baseline = per_mode_eval_losses(build_model(records, seed=seed), groups) + + telemetry_rows: list[dict] = [] + arms: dict[str, dict] = {} + for arm in ARMS: + model = train_arm( + arm, + groups, + records, + steps=steps, + lr=lr, + seed=seed, + telemetry_rows=telemetry_rows if arm == "wta" else None, + ) + per_mode = eval_per_mode(model, groups, baseline) + decode = eval_decode(model, groups) + arms[arm] = { + "per_mode": per_mode, + "decode": decode, + "coverage": per_mode["coverage"], + "collapse_indicator": per_mode["collapse_indicator"], + "training_metrics": model.last_training_metrics, + } + + telemetry_out.parent.mkdir(parents=True, exist_ok=True) + with telemetry_out.open("w", encoding="utf-8") as fh: + for row in telemetry_rows: + fh.write(json.dumps(row, sort_keys=True) + "\n") + + synthetic = run_synthetic_two_mode() + synthetic_checks = synthetic_assertions(synthetic) + + coverage_gain = arms["wta"]["coverage"] - arms["single_gold"]["coverage"] + semantic_regression = ( + arms["single_gold"]["decode"]["v2_verdict_rate"] + - arms["wta"]["decode"]["v2_verdict_rate"] + ) + coverage_ok = coverage_gain > 0.0 + semantic_ok = ( + semantic_regression + <= PREREGISTERED_THRESHOLDS["semantic_regression_max_points"] + ) + verdict = "wta_preserves_modes" if (coverage_ok and semantic_ok) else "rejected" + + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-314", + "question": ( + "Does winner-take-all (MCL) training with a preregistered floor " + "term retain more hard-valid canonical AST modes than single-gold " + "CE without a semantic regression, at equal budget?" + ), + "preregistered_thresholds": PREREGISTERED_THRESHOLDS, + "arms": ARMS, + "config": { + "steps": steps, + "lr": lr, + "seed": seed, + "n_prompts": len(groups), + "n_modes": sum(len(g.modes) for g in groups), + "dataset": str(DEFAULT_RESOURCE), + "dataset_manifest": dataset_manifest(groups), + "fixture_config_overrides": FIXTURE_CONFIG_OVERRIDES, + "torch_num_threads": 1, + }, + "synthetic_two_mode": { + "probabilities": synthetic, + "assertions": synthetic_checks, + "all_assertions_pass": all(synthetic_checks.values()), + }, + "arm_results": arms, + "comparison": { + "coverage_gain_wta_vs_single_gold": coverage_gain, + "semantic_regression_wta_vs_single_gold": semantic_regression, + "coverage_multi_gold": arms["multi_gold"]["coverage"], + }, + "threshold_checks": { + "coverage_ok": coverage_ok, + "semantic_ok": semantic_ok, + "synthetic_ok": all(synthetic_checks.values()), + }, + "telemetry_path": str(telemetry_out), + "verdict": verdict, + "wall_seconds": time.perf_counter() - start, + "honesty": ( + "Fixture-scale: 8 prompts x 2 modes is far below any ship-gate " + "prompt count — mechanism evidence only, not a production ship " + "claim. Mode identity is alpha-invariant canonical AST " + "fingerprints (never serialization strings); the frozen dataset " + "is hash-verified before training. All arms share model init, " + "steps, optimizer, per-step prompt coverage, and seeds. Decode is " + "deterministic beam search (unique valid ASTs per prompt == 1 by " + "construction); coverage is measured by deterministic per-mode " + "eval loss instead. The SLM-130 ambiguity sets are wiring reports " + "without a committed multi-mode corpus, so this dataset is " + "synthesized and frozen in-repo." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm314_winner_take_all", + "harness.experiments.slm299_edit_reachability", + ) + return payload + + +def render_markdown(payload: dict) -> str: + def f(x: float) -> str: + return f"{x:.3f}" + + lines = [ + "# SLM-314 (LAR2-05): winner-take-all over multi-mode gold", + "", + f"**Verdict: `{payload['verdict']}`** (fixture-scale matched arms; not a ship claim)", + "", + "## Preregistered (locked before results)", + "", + f"- arms: `{json.dumps(payload['arms'])}` — identical model init, steps, " + "optimizer, per-step prompt coverage, seeds", + f"- floor epsilon: {payload['preregistered_thresholds']['floor_epsilon']}; " + f"retained iff eval-loss improvement > " + f"{payload['preregistered_thresholds']['retained_improvement_min']} vs matched " + f"baseline and within {payload['preregistered_thresholds']['retention_gap_margin']} " + "of the prompt's best-improved mode", + f"- rule: {payload['preregistered_thresholds']['verdict_rule']}", + f"- frozen dataset: `{payload['config']['dataset']}` " + f"({payload['config']['n_prompts']} prompts x " + f"{payload['config']['n_modes'] // payload['config']['n_prompts']} modes, " + f"manifest `{payload['config']['dataset_manifest']['manifest_sha256'][:16]}…`)", + "", + "## Synthetic two-mode proof (deterministic)", + "", + "| arm | p(mode A) | p(mode B) | p(invalid hybrid) |", + "| --- | --- | --- | --- |", + ] + for arm, probs in payload["synthetic_two_mode"]["probabilities"].items(): + lines.append( + f"| {arm} | {f(probs['p_mode_a'])} | {f(probs['p_mode_b'])} | " + f"{f(probs['p_invalid_hybrid'])} |" + ) + checks = payload["synthetic_two_mode"]["assertions"] + lines += [ + "", + "Assertions: " + ", ".join(f"{k}={'PASS' if v else 'FAIL'}" for k, v in checks.items()), + "", + "## Matched arms (real tree-edit model, frozen multi-mode dataset)", + "", + "| arm | mode coverage | collapse | hard-valid | mode-hit | v2 verdict |", + "| --- | --- | --- | --- | --- | --- |", + ] + for arm in payload["arms"]: + r = payload["arm_results"][arm] + lines.append( + f"| {arm} | {f(r['coverage'])} | {f(r['collapse_indicator'])} | " + f"{f(r['decode']['hard_valid_rate'])} | {f(r['decode']['mode_hit_rate'])} | " + f"{f(r['decode']['v2_verdict_rate'])} |" + ) + cmp_ = payload["comparison"] + lines += [ + "", + f"Coverage gain (wta − single_gold): **{f(cmp_['coverage_gain_wta_vs_single_gold'])}**; " + f"v2 semantic regression: **{f(cmp_['semantic_regression_wta_vs_single_gold'])}** " + f"(budget 0.05). Multi-gold coverage: {f(cmp_['coverage_multi_gold'])}.", + "", + f"WTA per-step selected-mode telemetry: `{payload['telemetry_path']}`.", + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=5) + parser.add_argument("--lr", type=float, default=3e-3) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--dataset", type=Path, default=DEFAULT_RESOURCE) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + parser.add_argument("--telemetry-out", type=Path, default=DEFAULT_TELEMETRY_OUT) + args = parser.parse_args(argv) + + groups = read_frozen_dataset(args.dataset) + payload = build_report( + groups, + steps=args.steps, + lr=args.lr, + seed=args.seed, + telemetry_out=args.telemetry_out, + ) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"verdict={payload['verdict']} " + f"coverage_gain={payload['comparison']['coverage_gain_wta_vs_single_gold']:.3f} " + f"semantic_regression={payload['comparison']['semantic_regression_wta_vs_single_gold']:.3f} " + f"-> {args.json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm314_winner_take_all.py b/src/slm_training/harnesses/experiments/slm314_winner_take_all.py new file mode 100644 index 000000000..8c2045c90 --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm314_winner_take_all.py @@ -0,0 +1,547 @@ +"""SLM-314 (LAR2-05): winner-take-all (WTA/MCL) training over multi-mode gold. + +Problem: for many prompts there are several hard-valid canonical AST modes +(distinct verifier-accepted programs that all satisfy the prompt). Standard +single-gold CE picks ONE mode at the dataset level and gives zero gradient +toward the others (dataset-level mode collapse); duplicated-example +multi-gold CE averages the modes and can converge to an interpolated, +invalid hybrid. Winner-take-all / multiple-choice learning backpropagates +only the MINIMUM-loss mode per example, plus a small preregistered +coverage/floor term on the unused modes so they stay constrained instead of +drifting to zero probability. + +This module provides: + +- a deterministic, frozen fixture-scale multi-mode dataset (8 prompts x 2 + verifier-accepted canonical AST modes each), committed as a resource JSONL + with per-mode alpha-invariant canonical AST fingerprints (SLM-299 + ``_canonical_key`` sha256 via SLM-312 ``ast_fingerprint``) plus a sha256 + freeze manifest — mode identity is NEVER the serialization string; +- :func:`mode_supervision_loss` — the existing ``training_loss`` terms + computed deterministically per mode (dedicated seeded rng, model rng + restored afterwards); +- :func:`winner_take_all_loss` — per example, backpropagate + ``L_winner + floor_epsilon * mean(L_losers)`` with durable per-example + selected-mode + per-mode loss telemetry; +- :func:`run_synthetic_two_mode` — a minimal two-mode analytic fixture + proving single-gold CE collapses while WTA retains both modes (and + multi-gold CE produces invalid hybrids), with deterministic assertions + consumed by the tests and the CLI. + +Set-FTPO machinery is NOT reused (previously rejected). Torch is imported +lazily inside the loss/synthetic functions so dataset-freeze tests stay +cheap. +""" + +from __future__ import annotations + +import hashlib +import json +import random +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Sequence + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm312_state_sources import ( + ast_fingerprint, +) +from slm_training.models.tree_edit_diffusion import ( + _is_valid, + parse_statements, +) + +EXPERIMENT_ID = "slm314-winner-take-all" +DATASET_SCHEMA = "slm314-multimode-dataset/v1" +DEFAULT_RESOURCE = Path("src/slm_training/resources/data/slm314_multimode/modes.jsonl") + +# PREREGISTERED WTA floor weight (locked before any run; deviations are +# append-only and exploratory). The losing mode keeps probability mass +# ~ eps/(1+eps) per supervised factor at equilibrium instead of 0. +FLOOR_EPSILON = 0.1 + +# --------------------------------------------------------------------------- # +# Frozen multi-mode dataset +# --------------------------------------------------------------------------- # + +# (prompt, [(mode_source, placeholders), ...]) — every mode must parse through +# the real validator and have a DISTINCT canonical AST fingerprint. +MULTIMODE_PROGRAMS: list[tuple[str, list[tuple[str, list[str]]]]] = [ + ( + "hero with cta", + [ + ( + 'root = Stack([t, b], "column")\n' + 't = TextContent(":hero.title")\n' + 'b = Button(":cta.label")', + [":hero.title", ":cta.label"], + ), + ( + 'root = Stack([c], "column")\n' + 'c = Card([t, b], "column")\n' + 't = TextContent(":hero.title")\n' + 'b = Button(":cta.label")', + [":hero.title", ":cta.label"], + ), + ], + ), + ( + "simple text", + [ + ('root = Stack([t], "column")\nt = TextContent(":body")', [":body"]), + ('root = Stack([c], "column")\nc = Card([t], "column")\nt = TextContent(":body")', [":body"]), + ], + ), + ( + "form with email and submit", + [ + ( + 'root = Form([i, s], "column")\n' + 'i = TextContent(":form.email")\n' + 's = Button(":form.submit")', + [":form.email", ":form.submit"], + ), + ( + 'root = Form([c], "column")\n' + 'c = Card([i, s], "column")\n' + 'i = TextContent(":form.email")\n' + 's = Button(":form.submit")', + [":form.email", ":form.submit"], + ), + ], + ), + ( + "card with image", + [ + ( + 'root = Stack([c], "column")\n' + 'c = Card([im, t], "column")\n' + 'im = Image(":img.src")\n' + 't = TextContent(":img.caption")', + [":img.src", ":img.caption"], + ), + ( + 'root = Card([im, t], "column")\n' + 'im = Image(":img.src")\n' + 't = TextContent(":img.caption")', + [":img.src", ":img.caption"], + ), + ], + ), + ( + "two texts", + [ + ( + 'root = Stack([a, b], "column")\n' + 'a = TextContent(":title")\n' + 'b = TextContent(":subtitle")', + [":title", ":subtitle"], + ), + ( + 'root = Stack([c], "column")\n' + 'c = Card([a, b], "column")\n' + 'a = TextContent(":title")\n' + 'b = TextContent(":subtitle")', + [":title", ":subtitle"], + ), + ], + ), + ( + "button only", + [ + ('root = Stack([b], "column")\nb = Button(":cta")', [":cta"]), + ('root = Card([b], "column")\nb = Button(":cta")', [":cta"]), + ], + ), + ( + "nested card", + [ + ( + 'root = Stack([c], "column")\n' + 'c = Card([t], "column")\n' + 't = TextContent(":card.body")', + [":card.body"], + ), + ( + 'root = Stack([c], "column")\n' + 'c = Card([inner], "column")\n' + 'inner = Card([t], "column")\n' + 't = TextContent(":card.body")', + [":card.body"], + ), + ], + ), + ( + "form single field", + [ + ('root = Form([i], "column")\ni = TextContent(":q")', [":q"]), + ( + 'root = Form([c], "column")\n' + 'c = Card([i], "column")\n' + 'i = TextContent(":q")', + [":q"], + ), + ], + ), +] + + +@dataclass(frozen=True) +class ModeV1: + """One verifier-accepted canonical AST mode of a prompt.""" + + mode_index: int + source: str + canonical_fingerprint: str # alpha-invariant AST fingerprint (sha256) + placeholders: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["placeholders"] = list(self.placeholders) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "ModeV1": + return cls( + mode_index=int(data["mode_index"]), + source=str(data["source"]), + canonical_fingerprint=str(data["canonical_fingerprint"]), + placeholders=tuple(str(x) for x in data.get("placeholders") or ()), + ) + + +@dataclass(frozen=True) +class PromptGroupV1: + """One prompt with >=2 verifier-accepted canonical AST modes.""" + + group_id: str + prompt: str + modes: tuple[ModeV1, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "group_id": self.group_id, + "prompt": self.prompt, + "modes": [m.to_dict() for m in self.modes], + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "PromptGroupV1": + return cls( + group_id=str(data["group_id"]), + prompt=str(data["prompt"]), + modes=tuple(ModeV1.from_dict(m) for m in data.get("modes") or ()), + ) + + def records(self) -> list[ExampleRecord]: + """One ExampleRecord per mode (identical prompt, distinct gold).""" + return [ + ExampleRecord( + id=f"{self.group_id}__m{mode.mode_index}", + prompt=self.prompt, + openui=mode.source, + placeholders=list(mode.placeholders), + split="train", + ) + for mode in self.modes + ] + + +def _fingerprint_source(source: str) -> str: + """Alpha-invariant canonical AST fingerprint; raises when not valid.""" + if not _is_valid(source): + raise ValueError(f"mode is not verifier-accepted: {source!r}") + statements = parse_statements(source) + if statements is None: + raise ValueError(f"mode does not parse structurally: {source!r}") + return ast_fingerprint(statements) + + +def build_multimode_dataset() -> list[PromptGroupV1]: + """Deterministically construct the multi-mode dataset from the frozen spec. + + Fail-closed: every mode must be verifier-accepted through the real parser + and every prompt must have >=2 DISTINCT canonical AST fingerprints. + """ + groups: list[PromptGroupV1] = [] + for index, (prompt, mode_specs) in enumerate(MULTIMODE_PROGRAMS): + modes: list[ModeV1] = [] + seen: set[str] = set() + for mode_index, (source, placeholders) in enumerate(mode_specs): + fp = _fingerprint_source(source) + if fp in seen: + raise ValueError( + f"prompt {prompt!r} mode {mode_index} duplicates a canonical " + "AST fingerprint — modes must be distinct canonical ASTs" + ) + seen.add(fp) + modes.append( + ModeV1( + mode_index=mode_index, + source=source, + canonical_fingerprint=fp, + placeholders=tuple(placeholders), + ) + ) + if len(modes) < 2: + raise ValueError(f"prompt {prompt!r} has fewer than 2 modes") + groups.append( + PromptGroupV1( + group_id=f"mm_{index:02d}", prompt=prompt, modes=tuple(modes) + ) + ) + return groups + + +def _digest(obj: Any) -> str: + canonical = json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def dataset_manifest(groups: Sequence[PromptGroupV1]) -> dict[str, Any]: + row_hashes = [_digest(g.to_dict()) for g in groups] + return { + "dataset_schema": DATASET_SCHEMA, + "n_prompts": len(groups), + "n_modes": sum(len(g.modes) for g in groups), + "modes_per_prompt": sorted({len(g.modes) for g in groups}), + "row_sha256": row_hashes, + "manifest_sha256": _digest({"schema": DATASET_SCHEMA, "row_sha256": row_hashes}), + } + + +class FreezeIntegrityError(RuntimeError): + """A frozen multi-mode dataset row or manifest failed verification.""" + + +def freeze_dataset(groups: Sequence[PromptGroupV1], path: Path) -> dict[str, Any]: + """Write the dataset as JSONL plus a ``.manifest.json`` sidecar.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + for group in groups: + fh.write(json.dumps(group.to_dict(), sort_keys=True) + "\n") + manifest = dataset_manifest(groups) + path.with_suffix(path.suffix + ".manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return manifest + + +def read_frozen_dataset(path: Path) -> list[PromptGroupV1]: + """Load the frozen dataset, verifying row hashes + manifest (fail-closed).""" + manifest_path = path.with_suffix(path.suffix + ".manifest.json") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + groups: list[PromptGroupV1] = [] + with path.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line: + groups.append(PromptGroupV1.from_dict(json.loads(line))) + if [_digest(g.to_dict()) for g in groups] != list(manifest.get("row_sha256") or []): + raise FreezeIntegrityError(f"row hash mismatch against {manifest_path}") + if dataset_manifest(groups)["manifest_sha256"] != manifest.get("manifest_sha256"): + raise FreezeIntegrityError(f"manifest sha256 mismatch for {manifest_path}") + # Re-verify acceptance + fingerprint identity (never trust strings alone). + for group in groups: + for mode in group.modes: + if _fingerprint_source(mode.source) != mode.canonical_fingerprint: + raise FreezeIntegrityError( + f"{group.group_id} mode {mode.mode_index} fingerprint does not " + "match its re-parsed canonical AST" + ) + return groups + + +def single_gold_groups(groups: Sequence[PromptGroupV1]) -> list[PromptGroupV1]: + """Deterministic single-gold view: per prompt, the mode with the + lexicographically smallest canonical AST fingerprint.""" + reduced: list[PromptGroupV1] = [] + for group in groups: + winner = min(group.modes, key=lambda m: m.canonical_fingerprint) + reduced.append( + PromptGroupV1( + group_id=group.group_id, prompt=group.prompt, modes=(winner,) + ) + ) + return reduced + + +# --------------------------------------------------------------------------- # +# Per-mode deterministic supervision + WTA loss +# --------------------------------------------------------------------------- # + + +def _seed_int(*parts: str) -> int: + digest = hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest() + return int(digest[:16], 16) + + +def mode_supervision_loss(model: Any, record: ExampleRecord, *, seed: int) -> Any: + """The existing ``training_loss`` terms for ONE gold mode, computed with a + dedicated seeded rng (the model's own rng state is saved/restored) so the + per-mode loss is deterministic and identical across arms.""" + saved = model._rng + model._rng = random.Random(seed) + import torch + + try: + with torch.random.fork_rng(devices=[]): + torch.manual_seed(seed) + return model.training_loss([record]) + finally: + model._rng = saved + + +def select_winner(losses: Sequence[float]) -> int: + """Index of the minimum-loss mode; ties break to the lowest mode index + (deterministic).""" + if not losses: + raise ValueError("no mode losses") + best = 0 + for i, value in enumerate(losses): + if value < losses[best]: + best = i + return best + + +def winner_take_all_loss( + model: Any, + group: PromptGroupV1, + *, + floor_epsilon: float = FLOOR_EPSILON, + seed: int = 0, + eval_mode: bool = False, +) -> tuple[Any, dict[str, Any]]: + """WTA/MCL loss for one prompt group: backpropagate the minimum-loss mode + plus ``floor_epsilon * mean(losing-mode losses)`` (coverage/floor term). + + Returns ``(loss, telemetry)``; telemetry records the per-mode losses, the + selected winner fingerprint, and the floor weight (durable evidence). + With ``eval_mode=True`` no gradient-capable loss is required — the same + computation runs under ``torch.no_grad()`` by the caller. + """ + records = group.records() + losses: list[Any] = [] + detached: list[float] = [] + for mode, record in zip(group.modes, records): + loss = mode_supervision_loss( + model, + record, + seed=_seed_int( + "slm314", str(seed), group.group_id, mode.canonical_fingerprint + ), + ) + losses.append(loss) + detached.append(float(loss.detach().cpu())) + winner = select_winner(detached) + losers = [x for i, x in enumerate(losses) if i != winner] + total = losses[winner] + if losers and floor_epsilon > 0: + total = total + floor_epsilon * (sum(losers) / len(losers)) + telemetry = { + "group_id": group.group_id, + "floor_epsilon": floor_epsilon, + "winner_index": winner, + "winner_fingerprint": group.modes[winner].canonical_fingerprint, + "per_mode": [ + { + "mode_index": mode.mode_index, + "canonical_fingerprint": mode.canonical_fingerprint, + "loss": detached[i], + "selected": i == winner, + } + for i, mode in enumerate(group.modes) + ], + "total_loss": float(total.detach().cpu()), + } + return total, telemetry + + +# --------------------------------------------------------------------------- # +# Synthetic two-mode collapse-vs-retain fixture (analytic, deterministic) +# --------------------------------------------------------------------------- # + +# Two positions, two tokens each. Mode A = (0, 0) "Stack/column", mode +# B = (1, 1) "Card/row". Hybrids (0, 1) and (1, 0) are INVALID — the +# token-level average of the two modes. +SYNTHETIC_MODE_A = (0, 0) +SYNTHETIC_MODE_B = (1, 1) + +# PREREGISTERED synthetic assertions (locked before any run): +# - single-gold CE collapses: p(mode B) < SYNTHETIC_COLLAPSE_MAX +# - WTA retains both: p(mode B) >= SYNTHETIC_RETAINED_MIN and p(mode A) > p(B) +# - multi-gold CE averages: invalid-hybrid probability >= SYNTHETIC_HYBRID_MIN +SYNTHETIC_COLLAPSE_MAX = 0.002 +SYNTHETIC_RETAINED_MIN = 0.004 +SYNTHETIC_HYBRID_MIN = 0.3 + + +def run_synthetic_two_mode( + *, + steps: int = 400, + lr: float = 0.2, + floor_epsilon: float = FLOOR_EPSILON, +) -> dict[str, dict[str, float]]: + """Train a 2-position x 2-token softmax under the three arms. + + Full-batch, zero-init logits, no sampling — fully deterministic. Returns + per-arm sequence probabilities ``{p_mode_a, p_mode_b, p_invalid_hybrid}``. + """ + import torch + + def mode_loss(logits: Any, mode: tuple[int, int]) -> Any: + import torch.nn.functional as F + + return sum( + F.cross_entropy(logits[pos : pos + 1], torch.tensor([tok])) + for pos, tok in enumerate(mode) + ) + + def probs(logits: Any) -> dict[str, float]: + p = torch.softmax(logits, dim=-1) + pa = float(p[0, 0] * p[1, 0]) + pb = float(p[0, 1] * p[1, 1]) + hybrid = float(p[0, 0] * p[1, 1] + p[0, 1] * p[1, 0]) + return {"p_mode_a": pa, "p_mode_b": pb, "p_invalid_hybrid": hybrid} + + out: dict[str, dict[str, float]] = {} + for arm in ("single_gold", "multi_gold", "wta"): + torch.manual_seed(0) # identical init across arms (zero anyway) + logits = torch.zeros(2, 2, requires_grad=True) + opt = torch.optim.SGD([logits], lr=lr) + for _ in range(steps): + if arm == "single_gold": + loss = mode_loss(logits, SYNTHETIC_MODE_A) + elif arm == "multi_gold": + loss = 0.5 * ( + mode_loss(logits, SYNTHETIC_MODE_A) + + mode_loss(logits, SYNTHETIC_MODE_B) + ) + else: + la = mode_loss(logits, SYNTHETIC_MODE_A) + lb = mode_loss(logits, SYNTHETIC_MODE_B) + winner, loser = (la, lb) if float(la) <= float(lb) else (lb, la) + loss = winner + floor_epsilon * loser + opt.zero_grad() + loss.backward() + opt.step() + out[arm] = probs(logits.detach()) + return out + + +def synthetic_assertions(result: dict[str, dict[str, float]]) -> dict[str, bool]: + """Deterministic collapse-vs-retain assertions on the synthetic fixture.""" + return { + "single_gold_collapses": ( + result["single_gold"]["p_mode_b"] < SYNTHETIC_COLLAPSE_MAX + ), + "wta_retains_loser_mode": ( + result["wta"]["p_mode_b"] >= SYNTHETIC_RETAINED_MIN + ), + "wta_winner_dominant": ( + result["wta"]["p_mode_a"] > result["wta"]["p_mode_b"] + ), + "multi_gold_hybrid_invalid": ( + result["multi_gold"]["p_invalid_hybrid"] >= SYNTHETIC_HYBRID_MIN + ), + } diff --git a/src/slm_training/resources/data/slm314_multimode/modes.jsonl b/src/slm_training/resources/data/slm314_multimode/modes.jsonl new file mode 100644 index 000000000..c153e4ee6 --- /dev/null +++ b/src/slm_training/resources/data/slm314_multimode/modes.jsonl @@ -0,0 +1,8 @@ +{"group_id": "mm_00", "modes": [{"canonical_fingerprint": "80c5eda1a6896342816a9794926f6f9595c3c2b2f4a36b5beb68cb7fd91b3341", "mode_index": 0, "placeholders": [":hero.title", ":cta.label"], "source": "root = Stack([t, b], \"column\")\nt = TextContent(\":hero.title\")\nb = Button(\":cta.label\")"}, {"canonical_fingerprint": "6c454c79cac92994ad15f07e1e43019515d4df7a4dab2b09ab88a7133a0ba913", "mode_index": 1, "placeholders": [":hero.title", ":cta.label"], "source": "root = Stack([c], \"column\")\nc = Card([t, b], \"column\")\nt = TextContent(\":hero.title\")\nb = Button(\":cta.label\")"}], "prompt": "hero with cta"} +{"group_id": "mm_01", "modes": [{"canonical_fingerprint": "8c94a6a1e407ac005e58301670afa9aaccd0c2b1a04552693289f51a5a48ddf3", "mode_index": 0, "placeholders": [":body"], "source": "root = Stack([t], \"column\")\nt = TextContent(\":body\")"}, {"canonical_fingerprint": "8b2366208036fe324c2584d98e62cd697d1e0c2e1d2fc4461f19788d6510d691", "mode_index": 1, "placeholders": [":body"], "source": "root = Stack([c], \"column\")\nc = Card([t], \"column\")\nt = TextContent(\":body\")"}], "prompt": "simple text"} +{"group_id": "mm_02", "modes": [{"canonical_fingerprint": "4d0cafaed2d1d426605275844c7ffe9cbccb8c017cf18b7c4bff0932199f6fc8", "mode_index": 0, "placeholders": [":form.email", ":form.submit"], "source": "root = Form([i, s], \"column\")\ni = TextContent(\":form.email\")\ns = Button(\":form.submit\")"}, {"canonical_fingerprint": "29b4a54a4aa6b58ea3cd3ef25a8b282da7a08459de3959cd97443f1eec8e5d2d", "mode_index": 1, "placeholders": [":form.email", ":form.submit"], "source": "root = Form([c], \"column\")\nc = Card([i, s], \"column\")\ni = TextContent(\":form.email\")\ns = Button(\":form.submit\")"}], "prompt": "form with email and submit"} +{"group_id": "mm_03", "modes": [{"canonical_fingerprint": "5a798e31a8ce98a2a1a616a7e61531c632036bfc150d151bd775edc8903bff5b", "mode_index": 0, "placeholders": [":img.src", ":img.caption"], "source": "root = Stack([c], \"column\")\nc = Card([im, t], \"column\")\nim = Image(\":img.src\")\nt = TextContent(\":img.caption\")"}, {"canonical_fingerprint": "97b12398d55fd0836abcb35011122985efeb8dfbaf47d9d8d211f331aa598952", "mode_index": 1, "placeholders": [":img.src", ":img.caption"], "source": "root = Card([im, t], \"column\")\nim = Image(\":img.src\")\nt = TextContent(\":img.caption\")"}], "prompt": "card with image"} +{"group_id": "mm_04", "modes": [{"canonical_fingerprint": "b8528408c1ecc2292846532b492e323dec85034f4d98c445a1bfcb22d5120f58", "mode_index": 0, "placeholders": [":title", ":subtitle"], "source": "root = Stack([a, b], \"column\")\na = TextContent(\":title\")\nb = TextContent(\":subtitle\")"}, {"canonical_fingerprint": "9a2d5f9922a1612e18ac88aaed8a1c807f9d61f214ffab1787ede721e85db843", "mode_index": 1, "placeholders": [":title", ":subtitle"], "source": "root = Stack([c], \"column\")\nc = Card([a, b], \"column\")\na = TextContent(\":title\")\nb = TextContent(\":subtitle\")"}], "prompt": "two texts"} +{"group_id": "mm_05", "modes": [{"canonical_fingerprint": "13b7964bf2252bdbe82c5ce990ada9242baffb0f3b0b269be614ecefeedf6dd9", "mode_index": 0, "placeholders": [":cta"], "source": "root = Stack([b], \"column\")\nb = Button(\":cta\")"}, {"canonical_fingerprint": "1b9cd54f0cf1b9a5bac1f2c7bc8120ab78bccc0f7dc7c247705e065e7591c89a", "mode_index": 1, "placeholders": [":cta"], "source": "root = Card([b], \"column\")\nb = Button(\":cta\")"}], "prompt": "button only"} +{"group_id": "mm_06", "modes": [{"canonical_fingerprint": "de0096e262a8a20e11bb2338b2b99f3b38fca2034d657bf90c39ee7287d7d958", "mode_index": 0, "placeholders": [":card.body"], "source": "root = Stack([c], \"column\")\nc = Card([t], \"column\")\nt = TextContent(\":card.body\")"}, {"canonical_fingerprint": "54f7897efa57c43a2166b33229c3b1a8f043c020d84d78ef193c2aa4bfc0fd88", "mode_index": 1, "placeholders": [":card.body"], "source": "root = Stack([c], \"column\")\nc = Card([inner], \"column\")\ninner = Card([t], \"column\")\nt = TextContent(\":card.body\")"}], "prompt": "nested card"} +{"group_id": "mm_07", "modes": [{"canonical_fingerprint": "d60cd88bf91091b9ffc0087cf970ba229baa342ba8b955b32c4a906da8e881a3", "mode_index": 0, "placeholders": [":q"], "source": "root = Form([i], \"column\")\ni = TextContent(\":q\")"}, {"canonical_fingerprint": "c7f2a0d70546000767905aa498e7eaea65cdbfe66095ad6e9d93eab629e01969", "mode_index": 1, "placeholders": [":q"], "source": "root = Form([c], \"column\")\nc = Card([i], \"column\")\ni = TextContent(\":q\")"}], "prompt": "form single field"} diff --git a/src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json b/src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json new file mode 100644 index 000000000..32208cfff --- /dev/null +++ b/src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json @@ -0,0 +1,19 @@ +{ + "dataset_schema": "slm314-multimode-dataset/v1", + "manifest_sha256": "10d1edc939cc74c159858bf423aa3b457d691526efc41a125c50c5e97c7d42f9", + "modes_per_prompt": [ + 2 + ], + "n_modes": 16, + "n_prompts": 8, + "row_sha256": [ + "17bd182f4bddf979caabdf1248fe325418a4047575600ae6d10cad1c47314774", + "f6ac316055c35daefbdb864879ff504f269326d8d0bde64fc585b14d8be28882", + "015568fa00c3a12f895aeb58af0b85cb799e28f2c315d42fe02bfb573dbca89e", + "b22959ee91c136cbb9814a20ef9b4cf579924bab79c7c2f15fc3497e6ac9df62", + "35bd9d295405696d90b4748c671460a2db892552d824760b1a170ee56b436347", + "aad565646ad04affe14603d1eb0de57fed8d189e1ed2c7c360bf72ed8c78a5f6", + "708f07f6793b1832d3bee49b1f62fa2ab7ef3157b8de5f938c631888fa056b99", + "4b29cac7b71e9aff55a1914f674bc5f54656e01210bdcb6db714f34abccd73fd" + ] +} diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 1ec0d23fb..0edc43962 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -7859,6 +7859,26 @@ "note": "initial registration; SLM-312 LAR2-04 state sources: offline seedward oracle-guided valid intermediates + immutable content-addressed on-policy beam-trajectory snapshots labeled with the SLM-308 distance oracle and SLM-305 legal edits, fail-closed AST-fingerprint leak guards, matched {gold_only, seedward, on_policy, mixed} mixture arms with preregistered thresholds" } ] + }, + "harness.experiments.slm314_winner_take_all": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm314_winner_take_all.py", + "scripts/run_slm314_winner_take_all.py", + "tests/test_harnesses/experiments/test_slm314_winner_take_all.py", + "src/slm_training/resources/data/slm314_multimode/modes.jsonl", + "src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json", + "docs/design/iter-slm314-winner-take-all-20260724.json", + "docs/design/iter-slm314-winner-take-all-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-314 LAR2-05 winner-take-all (MCL): frozen 8-prompt x 2-mode verifier-accepted canonical-AST dataset (alpha-invariant fingerprints + sha256 manifest), min-loss-mode loss with preregistered 0.1 coverage/floor term, deterministic two-mode synthetic collapse-vs-retain proof, matched {single_gold, multi_gold, wta} arms" + } + ] } } } diff --git a/tests/test_harnesses/experiments/test_slm314_winner_take_all.py b/tests/test_harnesses/experiments/test_slm314_winner_take_all.py new file mode 100644 index 000000000..82be75e7c --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm314_winner_take_all.py @@ -0,0 +1,244 @@ +"""SLM-314 (LAR2-05): multi-mode dataset freeze, WTA loss, synthetic fixture.""" + +from __future__ import annotations + +import json + +import pytest + +# Tiny CPU fixture models: torch thread thrash dominates otherwise. +import torch + +torch.set_num_threads(1) + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm314_winner_take_all import ( + DEFAULT_RESOURCE, + FLOOR_EPSILON, + FreezeIntegrityError, + PromptGroupV1, + build_multimode_dataset, + dataset_manifest, + freeze_dataset, + mode_supervision_loss, + read_frozen_dataset, + run_synthetic_two_mode, + select_winner, + single_gold_groups, + synthetic_assertions, + winner_take_all_loss, + _fingerprint_source, +) +from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, +) + +_SMALL_CONFIG = { + "context_backend": "scratch", + "seed": 0, + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 2, + "max_search_steps": 4, + "beam_width": 2, + "expand_per_state": 2, +} + +MODE_A = 'root = Stack([t], "column")\nt = TextContent(":body")' +MODE_B = 'root = Stack([c], "column")\nc = Card([t], "column")\nt = TextContent(":body")' + + +def _group() -> PromptGroupV1: + return next(g for g in build_multimode_dataset() if g.group_id == "mm_01") + + +def _model(records: list[ExampleRecord]) -> TreeEditDiffusionModel: + torch.manual_seed(0) + return TreeEditDiffusionModel.from_records( + records, config=TreeEditDiffusionConfig(**_SMALL_CONFIG), device="cpu" + ) + + +# --- mode identity via canonical fingerprints ------------------------------- + + +def test_fingerprint_alpha_invariant_same_ast() -> None: + """Two serializations of the same AST (renamed statements, reordered + declarations) share one canonical fingerprint.""" + renamed = 'root = Stack([x1], "column")\nx1 = TextContent(":body")' + reordered = 't = TextContent(":body")\nroot = Stack([t], "column")' + assert _fingerprint_source(MODE_A) == _fingerprint_source(renamed) + assert _fingerprint_source(MODE_A) == _fingerprint_source(reordered) + + +def test_fingerprint_distinct_for_different_asts() -> None: + assert _fingerprint_source(MODE_A) != _fingerprint_source(MODE_B) + + +def test_fingerprint_requires_verifier_acceptance() -> None: + with pytest.raises(ValueError): + _fingerprint_source("root = NotAComponent([t])") + + +# --- frozen dataset ---------------------------------------------------------- + + +def test_dataset_every_prompt_has_two_distinct_valid_modes() -> None: + groups = build_multimode_dataset() + assert len(groups) == 8 + for group in groups: + assert len(group.modes) >= 2 + fps = [m.canonical_fingerprint for m in group.modes] + assert len(set(fps)) == len(fps) + for mode in group.modes: + # fingerprint identity is recomputed through the real parser + assert _fingerprint_source(mode.source) == mode.canonical_fingerprint + + +def test_committed_resource_matches_builder_and_manifest() -> None: + frozen = read_frozen_dataset(DEFAULT_RESOURCE) + live = build_multimode_dataset() + assert [g.to_dict() for g in frozen] == [g.to_dict() for g in live] + assert dataset_manifest(frozen)["n_modes"] == 16 + + +def test_freeze_roundtrip_and_tamper_detection(tmp_path) -> None: + path = tmp_path / "modes.jsonl" + groups = build_multimode_dataset() + manifest = freeze_dataset(groups, path) + assert dataset_manifest(read_frozen_dataset(path)) == manifest + # Tamper with one mode source: fail-closed on reload. + rows = [json.loads(line) for line in path.read_text().splitlines()] + rows[0]["modes"][0]["source"] = MODE_B + path.write_text("\n".join(json.dumps(r, sort_keys=True) for r in rows) + "\n") + with pytest.raises(FreezeIntegrityError): + read_frozen_dataset(path) + + +def test_single_gold_groups_deterministic() -> None: + groups = build_multimode_dataset() + reduced = single_gold_groups(groups) + assert all(len(g.modes) == 1 for g in reduced) + for full, one in zip(groups, reduced): + expected = min(m.canonical_fingerprint for m in full.modes) + assert one.modes[0].canonical_fingerprint == expected + + +# --- WTA loss ---------------------------------------------------------------- + + +def test_select_winner_min_loss_tie_breaks_lowest() -> None: + assert select_winner([2.0, 1.0, 3.0]) == 1 + assert select_winner([1.0, 1.0]) == 0 + with pytest.raises(ValueError): + select_winner([]) + + +def test_wta_selects_min_loss_mode_and_records_telemetry() -> None: + group = _group() + records = [r for g in build_multimode_dataset() for r in g.records()] + model = _model(records) + loss, telemetry = winner_take_all_loss(model, group, seed=0) + losses = [m["loss"] for m in telemetry["per_mode"]] + winner = telemetry["winner_index"] + assert winner == select_winner(losses) + assert telemetry["winner_fingerprint"] == ( + group.modes[winner].canonical_fingerprint + ) + assert sum(1 for m in telemetry["per_mode"] if m["selected"]) == 1 + # total = winner + eps * mean(losers) + losers = [x for i, x in enumerate(losses) if i != winner] + expected = losses[winner] + FLOOR_EPSILON * (sum(losers) / len(losers)) + assert telemetry["total_loss"] == pytest.approx(expected, rel=1e-5) + assert float(loss.detach()) == pytest.approx(expected, rel=1e-5) + + +def test_wta_floor_term_backpropagates_into_losing_mode() -> None: + """The coverage/floor term must constrain the losing mode: gradient flows + into both modes' losses (winner fully, loser with the epsilon weight).""" + la = torch.tensor(2.0, requires_grad=True) + lb = torch.tensor(3.0, requires_grad=True) + losses = [la, lb] + winner = select_winner([2.0, 3.0]) + losers = [x for i, x in enumerate(losses) if i != winner] + total = losses[winner] + FLOOR_EPSILON * (sum(losers) / len(losers)) + total.backward() + assert la.grad.item() == pytest.approx(1.0) + assert lb.grad.item() == pytest.approx(FLOOR_EPSILON) + + +def test_mode_supervision_loss_deterministic_and_rng_restored() -> None: + group = _group() + records = [r for g in build_multimode_dataset() for r in g.records()] + model = _model(records) + record = group.records()[0] + rng_state_before = model._rng.getstate() + l1 = float(mode_supervision_loss(model, record, seed=123).detach()) + l2 = float(mode_supervision_loss(model, record, seed=123).detach()) + assert l1 == pytest.approx(l2) + assert model._rng.getstate() == rng_state_before + + +def test_winner_take_all_loss_deterministic() -> None: + group = _group() + records = [r for g in build_multimode_dataset() for r in g.records()] + m1 = _model(records) + m2 = _model(records) + l1, t1 = winner_take_all_loss(m1, group, seed=7) + l2, t2 = winner_take_all_loss(m2, group, seed=7) + assert t1 == t2 + assert float(l1.detach()) == pytest.approx(float(l2.detach())) + + +# --- synthetic two-mode collapse-vs-retain fixture --------------------------- + + +def test_synthetic_single_gold_collapses_wta_retains() -> None: + result = run_synthetic_two_mode() + checks = synthetic_assertions(result) + assert all(checks.values()), checks + # Explicit collapse/retain numbers (preregistered thresholds). + assert result["single_gold"]["p_mode_b"] < 0.002 + assert result["wta"]["p_mode_b"] >= 0.004 + assert result["wta"]["p_mode_a"] > result["wta"]["p_mode_b"] + # Averaging arm puts >=30% mass on invalid hybrids. + assert result["multi_gold"]["p_invalid_hybrid"] >= 0.3 + + +def test_synthetic_deterministic() -> None: + assert run_synthetic_two_mode() == run_synthetic_two_mode() + + +def test_synthetic_floor_controls_retained_mass() -> None: + """Larger floor epsilon -> more losing-mode mass retained (monotone).""" + low = run_synthetic_two_mode(floor_epsilon=0.05) + high = run_synthetic_two_mode(floor_epsilon=0.3) + assert high["wta"]["p_mode_b"] > low["wta"]["p_mode_b"] + + +# --- arm isolation ------------------------------------------------------------ + + +def test_arm_isolation_identical_budgets() -> None: + """The CLI's train_arm gives every arm the same model init, step count, + and per-step prompt coverage; only the loss weighting differs.""" + from scripts.run_slm314_winner_take_all import ARMS, FIXTURE_CONFIG_OVERRIDES + + assert ARMS == ("single_gold", "multi_gold", "wta") + # Same declared fixture config object for all arms (no per-arm overrides). + assert set(FIXTURE_CONFIG_OVERRIDES) >= {"d_model", "denoiser_layers"} + groups = build_multimode_dataset() + records = [r for g in groups for r in g.records()] + torch.manual_seed(0) + m1 = TreeEditDiffusionModel.from_records( + records, config=TreeEditDiffusionConfig(**_SMALL_CONFIG), device="cpu" + ) + torch.manual_seed(0) + m2 = TreeEditDiffusionModel.from_records( + records, config=TreeEditDiffusionConfig(**_SMALL_CONFIG), device="cpu" + ) + s1 = {k: v.detach().clone() for k, v in m1.state_dict().items()} + s2 = m2.state_dict() + assert all(torch.equal(s1[k], s2[k]) for k in s1) From 48e5cadc13f9e6e431491da615badb4a0614c17a Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:24:37 -0500 Subject: [PATCH 09/10] SLM-317: do-no-harm AR->repair hybrid + advancement screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Metamorphic invariance (alpha-rename, statement reorder, formatting, AST roundtrip) tested before model evaluation. - Hard do-no-harm commit rule: COMMIT on deterministic hard-evidence improvement or calibrated soft improvement without hard regression; RETAIN on regression/no-improvement; ABSTAIN on missing candidate; source + candidate + exact reason durable per example; oracle commit selector as non-promotable upper bound. - Matched arms {ar_only, repair_only, ar_repair_historical, ar_repair_improved, oracle_commit} on the frozen SLM-155 decision corpus with paired per-example outcomes and explicit invalid-over-valid counts per arm. - Gates: Safety PASS (iov=0), Value FAIL (0/16 improvements, Wilson [0.0, 0.194] vs 0.05), Reachability PASS. Disposition: inconclusive — LAR3 remains open but NOT advanced; powered rerun required. - 20 new tests; slm317 component v1. --- .../iter-slm317-repair-hybrid-20260724.json | 899 ++++++++++++++++++ .../iter-slm317-repair-hybrid-20260724.md | 53 ++ scripts/run_slm317_repair_hybrid.py | 631 ++++++++++++ .../experiments/slm317_repair_hybrid.py | 446 +++++++++ src/slm_training/resources/versions.json | 18 + .../experiments/test_slm317_repair_hybrid.py | 263 +++++ 6 files changed, 2310 insertions(+) create mode 100644 docs/design/iter-slm317-repair-hybrid-20260724.json create mode 100644 docs/design/iter-slm317-repair-hybrid-20260724.md create mode 100644 scripts/run_slm317_repair_hybrid.py create mode 100644 src/slm_training/harnesses/experiments/slm317_repair_hybrid.py create mode 100644 tests/test_harnesses/experiments/test_slm317_repair_hybrid.py diff --git a/docs/design/iter-slm317-repair-hybrid-20260724.json b/docs/design/iter-slm317-repair-hybrid-20260724.json new file mode 100644 index 000000000..eddd68faf --- /dev/null +++ b/docs/design/iter-slm317-repair-hybrid-20260724.json @@ -0,0 +1,899 @@ +{ + "experiment": "slm317-repair-hybrid", + "issue": "SLM-317", + "question": "Does a do-no-harm AR\u2192repair hybrid (deterministic commit rule) improve paired hard-valid outcomes over AR only without ever damaging a valid AR program?", + "preregistered": { + "min_useful_effect": 0.05, + "safety_rule": "invalid-over-valid count of ar_repair_improved vs ar_only on the frozen safety set (all eval examples) must be exactly 0", + "value_rule": "paired hard-valid improvement of ar_repair_improved over ar_only: Wilson 95% lower bound of improvements/n_paired must exceed 0.05 with zero damages", + "disposition_rule": "repair_positive iff safety AND value AND reachability gates pass; repair_negative iff safety fails OR the value Wilson UPPER bound is below the minimum useful effect (effect ruled out); else inconclusive", + "historical_deviation": "pre-SLM-305 4-action edit space is not reproducible on this branch (edit space extended in place, no legacy-subset knob); the historical arm keeps mutation_count value labels + legacy STOP-slot accounting only" + }, + "arms": [ + "ar_only", + "repair_only", + "ar_repair_historical", + "ar_repair_improved", + "oracle_commit" + ], + "promotable_arms": [ + "ar_only", + "ar_repair_historical", + "ar_repair_improved", + "repair_only" + ], + "config": { + "n_train_decisions": 16, + "n_eval_decisions": 8, + "seeds": [ + 0, + 1 + ], + "steps": 6, + "lr": 0.003, + "scorer_steps": 20, + "fixture_config_overrides": { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 3, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3 + }, + "torch_num_threads": 1 + }, + "arm_summaries": { + "ar_only": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "repair_only": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "ar_repair_historical": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "ar_repair_improved": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "oracle_commit": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + } + }, + "paired_vs_ar_only": { + "repair_only": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + }, + "ar_repair_historical": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + }, + "ar_repair_improved": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + }, + "oracle_commit": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + } + }, + "invalid_over_valid_per_arm": { + "repair_only": 0, + "ar_repair_historical": 0, + "ar_repair_improved": 0, + "oracle_commit": 0 + }, + "commit_reason_counts": { + "ar_repair_historical": { + "hard_regression": 16 + }, + "ar_repair_improved": { + "no_improvement": 16 + }, + "oracle_commit": { + "oracle_no_gain": 16 + } + }, + "gates": { + "safety": { + "rule": "invalid-over-valid count of ar_repair_improved vs ar_only on the frozen safety set (all eval examples) must be exactly 0", + "invalid_over_valid": 0, + "pass": true + }, + "value": { + "rule": "paired hard-valid improvement of ar_repair_improved over ar_only: Wilson 95% lower bound of improvements/n_paired must exceed 0.05 with zero damages", + "improvements": 0, + "n_paired": 16, + "wilson": { + "n": 16, + "estimate": 0.0, + "low": 0.0, + "high": 0.19360768053443644, + "confidence_level": 0.95 + }, + "min_useful_effect": 0.05, + "pass": false + }, + "reachability": { + "artifacts": { + "docs/design/iter-slm299-edit-reachability-20260724.json": true, + "docs/design/iter-slm299-edit-reachability-20260724.md": true, + "docs/design/iter-slm291-evidence-bundles-20260724.json": true, + "docs/design/iter-slm291-evidence-bundles-20260724.md": true + }, + "pass": true + } + }, + "wilson_damage": { + "n": 16, + "estimate": 0.0, + "low": 0.0, + "high": 0.19360768053443644, + "confidence_level": 0.95 + }, + "oracle_upper_bound": { + "valid_rate": 1.0, + "sanity": true + }, + "per_example_outcomes": [ + { + "record_id": "decision-0", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-0", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-1", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-1", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-2", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-2", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-3", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-3", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-4", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-4", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-5", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-5", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-6", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-6", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-7", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-7", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-0", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-0", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-1", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-1", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-2", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-2", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-3", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-3", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-4", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-4", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-5", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-5", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-6", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-6", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-7", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-7", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + } + ], + "decisions_path": "outputs/slm317/commit_decisions.jsonl", + "disposition": "inconclusive", + "lar3_statement": "LAR3 (learned repair integration) remains OPEN but NOT advanced: the screen neither cleared the value gate nor ruled the effect out at fixture power; a powered rerun is required before LAR3 can open or close.", + "wall_seconds": 67.08616952600278, + "honesty": "Fixture-scale wiring screen: the frozen corpus is the SLM-155 synthetic decision fixture (n eval decisions below any ship-gate prompt count), models are tiny CPU fixtures trained for a handful of steps, and no ship-gate claim is made. The commit rule is deterministic and metamorphism-invariant by construction; the screen measures the mechanism, not production quality. The historical arm's pre-SLM-305 4-action space is declared non-reproducible on this branch (see preregistered deviation).", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "f6b173105eb2f3e2e5fa2b5fea7e3af452cc2b83", + "code_dirty": true, + "components": { + "harness.experiments.slm317_repair_hybrid": "v1", + "harness.experiments.slm155_factorization_comparison": "v2", + "harness.experiments.slm299_edit_reachability": "v4" + }, + "stamped_at": "2026-07-25T00:11:13.905812+00:00" + } +} diff --git a/docs/design/iter-slm317-repair-hybrid-20260724.md b/docs/design/iter-slm317-repair-hybrid-20260724.md new file mode 100644 index 000000000..e7e2d5e05 --- /dev/null +++ b/docs/design/iter-slm317-repair-hybrid-20260724.md @@ -0,0 +1,53 @@ +# SLM-317 (LAR2-06): do-no-harm AR→repair hybrid screen + +**Disposition: `inconclusive`** — LAR3 (learned repair integration) remains OPEN but NOT advanced: the screen neither cleared the value gate nor ruled the effect out at fixture power; a powered rerun is required before LAR3 can open or close. + +Fixture-scale mechanism screen; **not a ship claim**. + +## Preregistered (locked before results) + +- safety: invalid-over-valid count of ar_repair_improved vs ar_only on the frozen safety set (all eval examples) must be exactly 0 +- value: paired hard-valid improvement of ar_repair_improved over ar_only: Wilson 95% lower bound of improvements/n_paired must exceed 0.05 with zero damages +- disposition: repair_positive iff safety AND value AND reachability gates pass; repair_negative iff safety fails OR the value Wilson UPPER bound is below the minimum useful effect (effect ruled out); else inconclusive +- declared deviation: pre-SLM-305 4-action edit space is not reproducible on this branch (edit space extended in place, no legacy-subset knob); the historical arm keeps mutation_count value labels + legacy STOP-slot accounting only + +## Arms (matched examples + seeds + budgets) + +| arm | n | hard-valid rate | mean hard rank | invalid-over-valid vs AR | +| --- | --- | --- | --- | --- | +| ar_only | 16 | 1.000 | 3.000 | — | +| repair_only | 16 | 1.000 | 3.000 | 0 | +| ar_repair_historical | 16 | 1.000 | 3.000 | 0 | +| ar_repair_improved | 16 | 1.000 | 3.000 | 0 | +| oracle_commit | 16 | 1.000 | 3.000 | 0 | + +## Paired outcomes vs `ar_only` (per-example, nothing aggregated away) + +| arm | both valid | AR valid, arm invalid | AR invalid, arm valid | both invalid | unpaired | +| --- | --- | --- | --- | --- | --- | +| repair_only | 16 | 0 | 0 | 0 | 0 | +| ar_repair_historical | 16 | 0 | 0 | 0 | 0 | +| ar_repair_improved | 16 | 0 | 0 | 0 | 0 | +| oracle_commit | 16 | 0 | 0 | 0 | 0 | + +## Advancement gates + +- **Safety**: invalid-over-valid = 0 (must be 0) → PASS +- **Value**: improvements 0/16, Wilson 95% [0.0, 0.19360768053443644] vs minimum useful effect 0.05 → FAIL +- **Reachability/provenance**: slm299/291 artifacts present → PASS + +## Commit reasons + +| arm | reason | count | +| --- | --- | --- | +| ar_repair_historical | hard_regression | 16 | +| ar_repair_improved | no_improvement | 16 | +| oracle_commit | oracle_no_gain | 16 | + +Oracle commit upper bound hard-valid rate: 1.000 (sanity ≥ improved hybrid: True). + +Durable per-example commit decisions: `outputs/slm317/commit_decisions.jsonl`. + +## Honesty + +Fixture-scale wiring screen: the frozen corpus is the SLM-155 synthetic decision fixture (n eval decisions below any ship-gate prompt count), models are tiny CPU fixtures trained for a handful of steps, and no ship-gate claim is made. The commit rule is deterministic and metamorphism-invariant by construction; the screen measures the mechanism, not production quality. The historical arm's pre-SLM-305 4-action space is declared non-reproducible on this branch (see preregistered deviation). diff --git a/scripts/run_slm317_repair_hybrid.py b/scripts/run_slm317_repair_hybrid.py new file mode 100644 index 000000000..932216a88 --- /dev/null +++ b/scripts/run_slm317_repair_hybrid.py @@ -0,0 +1,631 @@ +#!/usr/bin/env python3 +"""SLM-317 (LAR2-06): valid-state repair advancement screen (AR→repair hybrid). + +Runs the preregistered do-no-harm screen over matched arms on the frozen +SLM-155 fixture decision corpus (identical examples + seeds + budgets): + +- ``ar_only`` — the SLM-155 AR legal-action path; final = AR program. +- ``repair_only`` — X22 value-guided beam decode from the minimal seed. +- ``ar_repair_historical`` — AR source → repair decode under the pre-LAR2 + config (``mutation_count`` value labels, legacy STOP-slot accounting). The + pre-SLM-305 4-action edit space is NOT reproducible on this branch (the + space was extended in place; no legacy-subset knob exists) — declared + deviation, the arm keeps the historical value/stop config only. +- ``ar_repair_improved`` — AR source → repair decode under this branch's + config (``bounded_distance`` oracle value labels + corrected STOP + accounting), then the do-no-harm commit rule with the policy value head as + the calibrated learned score. +- ``oracle_commit`` — commits exactly when the candidate's hard evidence is + strictly better (non-promotable upper bound). + +Every final is scored on the LAR1-02 unified ladder (official parse → output +contract → meaningful_program_v1). Paired per-example outcomes key on +(record_id, seed); invalid-over-valid counts are reported per arm and never +aggregated away. Advancement gates: Safety = 0 invalid-over-valid on the +frozen safety set; Value = paired improvement over AR with Wilson lower bound +above the preregistered minimum useful effect (0.05); Reachability/provenance += LAR2 artifacts present (slm299 reachability, slm291 evidence bundles). +Exactly one disposition: repair_positive | repair_negative | inconclusive, +with an explicit LAR3 open/close statement. + +Writes ``docs/design/iter-slm317-repair-hybrid-20260724.{json,md}`` and +durable commit decisions under ``outputs/slm317/``. + +Example: + python -m scripts.run_slm317_repair_hybrid --steps 8 +""" + +from __future__ import annotations + +import argparse +import json +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from slm_training.evals.power_protocol import wilson_interval +from slm_training.harnesses.experiments.slm155_factorization_comparison import ( + _program_from_actions, + paired_validity_table, +) +from slm_training.harnesses.experiments.slm317_repair_hybrid import ( + DISPOSITION_INCONCLUSIVE, + DISPOSITION_NEGATIVE, + DISPOSITION_POSITIVE, + EXPERIMENT_ID, + CommitDecision, + commit, + hard_evidence, + oracle_commit, + repair_decode, + state_value, +) +from slm_training.models.legal_action_scorer import ( + LegalActionScorerConfig, + make_fixture_decisions, + train_fixture_scorer, +) +from slm_training.versioning import build_version_stamp + +DEFAULT_JSON_OUT = Path("docs/design/iter-slm317-repair-hybrid-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm317-repair-hybrid-20260724.md") +DEFAULT_DECISIONS_OUT = Path("outputs/slm317/commit_decisions.jsonl") + +ARMS = ( + "ar_only", + "repair_only", + "ar_repair_historical", + "ar_repair_improved", + "oracle_commit", +) +PROMOTABLE_ARMS = {"ar_only", "repair_only", "ar_repair_historical", "ar_repair_improved"} + +# LAR2 reachability/provenance artifacts this screen builds on. +REACHABILITY_ARTIFACTS = ( + "docs/design/iter-slm299-edit-reachability-20260724.json", + "docs/design/iter-slm299-edit-reachability-20260724.md", + "docs/design/iter-slm291-evidence-bundles-20260724.json", + "docs/design/iter-slm291-evidence-bundles-20260724.md", +) + +# PREREGISTERED (locked before any run; deviations append-only/exploratory). +PREREGISTERED = { + "min_useful_effect": 0.05, + "safety_rule": ( + "invalid-over-valid count of ar_repair_improved vs ar_only on the " + "frozen safety set (all eval examples) must be exactly 0" + ), + "value_rule": ( + "paired hard-valid improvement of ar_repair_improved over ar_only: " + "Wilson 95% lower bound of improvements/n_paired must exceed 0.05 " + "with zero damages" + ), + "disposition_rule": ( + "repair_positive iff safety AND value AND reachability gates pass; " + "repair_negative iff safety fails OR the value Wilson UPPER bound is " + "below the minimum useful effect (effect ruled out); else inconclusive" + ), + "historical_deviation": ( + "pre-SLM-305 4-action edit space is not reproducible on this branch " + "(edit space extended in place, no legacy-subset knob); the " + "historical arm keeps mutation_count value labels + legacy STOP-slot " + "accounting only" + ), +} + +FIXTURE_CONFIG_OVERRIDES = { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 3, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3, +} + + +@dataclass(frozen=True) +class ArmOutcome: + """Paired per-example final outcome for one arm (duck-types slm155's).""" + + record_id: str + arm_id: str + seed: int + final_source: str + v1_verdict: bool + hard_rank: int + commit_reason: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "record_id": self.record_id, + "arm_id": self.arm_id, + "seed": self.seed, + "final_source": self.final_source, + "v1_verdict": self.v1_verdict, + "hard_rank": self.hard_rank, + "commit_reason": self.commit_reason, + } + + +def _inventory_for(source: str) -> list[str]: + import re + + slots = sorted(set(re.findall(r":[A-Za-z][\w.]*", source))) + return slots or [":content"] + + +def _prompt_for(decision_id: str) -> str: + return f"fixture decision {decision_id}: render the accepted content" + + +def _build_model(records, *, seed: int, value_label_mode: str, stop_accounting: str): + import torch + + from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + ) + + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + seed=seed, + value_label_mode=value_label_mode, + stop_slot_accounting=stop_accounting, + context_backend="scratch", + **FIXTURE_CONFIG_OVERRIDES, + ) + return TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + + +def _train(model, records, *, steps: int, lr: float): + import torch + + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=lr) + model.train() + for _ in range(steps): + loss = model.training_loss(records) + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +def run_screen( + *, + n_train: int, + n_eval: int, + seeds: tuple[int, ...], + steps: int, + lr: float, + scorer_steps: int, + decisions_out: Path, +) -> dict[str, Any]: + import torch + + from slm_training.dsl.schema import ExampleRecord + + torch.set_num_threads(1) # tiny CPU fixture; identical for every arm + start = time.perf_counter() + + train_decisions = make_fixture_decisions(n=n_train, seed=0) + eval_decisions = make_fixture_decisions(n=n_eval, seed=1) + scorer_result = train_fixture_scorer( + train_decisions, + config=LegalActionScorerConfig(variant="mlp", seed=0), + steps=scorer_steps, + lr=0.05, + ) + scorer = scorer_result["scorer"] + + def ar_source_for(decision) -> str: + scores = scorer.score( + decision.context_features, + decision.state_features, + decision.legal_actions, + plan_features=decision.plan_features, + plan_action_features=decision.plan_action_features, + pack_id=decision.pack_id, + ) + chosen = scorer.decode(scores, decision.legal_actions).action_identity + return _program_from_actions([chosen] if chosen else []) + + train_records = [ + ExampleRecord( + id=d.decision_id, + prompt=_prompt_for(d.decision_id), + openui=_program_from_actions( + [d.accepted_action_ids[0]] if d.accepted_action_ids else [] + ), + placeholders=_inventory_for( + _program_from_actions( + [d.accepted_action_ids[0]] if d.accepted_action_ids else [] + ) + ), + ) + for d in train_decisions + ] + + model_hist = _train( + _build_model( + train_records, + seed=0, + value_label_mode="mutation_count", + stop_accounting="legacy", + ), + train_records, + steps=steps, + lr=lr, + ) + model_imp = _train( + _build_model( + train_records, + seed=0, + value_label_mode="bounded_distance", + stop_accounting="corrected", + ), + train_records, + steps=steps, + lr=lr, + ) + + outcomes: list[ArmOutcome] = [] + commit_rows: list[CommitDecision] = [] + + def record(arm: str, record_id: str, seed: int, final: str, reason: str | None): + ev = hard_evidence(final) + outcomes.append( + ArmOutcome( + record_id=record_id, + arm_id=arm, + seed=seed, + final_source=final, + v1_verdict=ev.v1_verdict, + hard_rank=ev.rank, + commit_reason=reason, + ) + ) + + for seed in seeds: + for decision in eval_decisions: + rid = decision.decision_id + prompt = _prompt_for(rid) + ar_source = ar_source_for(decision) + inventory = _inventory_for(ar_source) + + # Arm 1: AR only. + record("ar_only", rid, seed, ar_source, None) + + # Arm 2: repair-only from the minimal seed (improved model decode). + ctx, ctx_pad = model_imp._encode_context( + [model_imp._format_context(prompt, slot_contract=inventory)] + ) + seed_final, _ = model_imp._decode_one(ctx, ctx_pad, inventory) + record("repair_only", rid, seed, seed_final, None) + + # Arm 3: AR → historical repair (pre-LAR2 value/stop config). + cand_hist, _ = repair_decode(model_hist, ar_source, inventory, prompt) + decision_hist = commit( + ar_source, + cand_hist, + evidence={ + "learned_score_source": state_value( + model_hist, ar_source, inventory, prompt + ), + "learned_score_candidate": state_value( + model_hist, cand_hist, inventory, prompt + ), + }, + record_id=rid, + arm_id="ar_repair_historical", + seed=seed, + ) + commit_rows.append(decision_hist) + record( + "ar_repair_historical", rid, seed, decision_hist.final, decision_hist.reason + ) + + # Arm 4: AR → improved repair (this branch's config). + cand_imp, _ = repair_decode(model_imp, ar_source, inventory, prompt) + decision_imp = commit( + ar_source, + cand_imp, + evidence={ + "learned_score_source": state_value( + model_imp, ar_source, inventory, prompt + ), + "learned_score_candidate": state_value( + model_imp, cand_imp, inventory, prompt + ), + }, + record_id=rid, + arm_id="ar_repair_improved", + seed=seed, + ) + commit_rows.append(decision_imp) + record( + "ar_repair_improved", rid, seed, decision_imp.final, decision_imp.reason + ) + + # Arm 5: oracle commit selector (non-promotable upper bound). + decision_oracle = oracle_commit( + ar_source, cand_imp, record_id=rid, arm_id="oracle_commit", seed=seed + ) + commit_rows.append(decision_oracle) + record("oracle_commit", rid, seed, decision_oracle.final, decision_oracle.reason) + + decisions_out.parent.mkdir(parents=True, exist_ok=True) + with decisions_out.open("w", encoding="utf-8") as fh: + for row in commit_rows: + fh.write(json.dumps(row.to_dict(), sort_keys=True) + "\n") + + by_arm: dict[str, list[ArmOutcome]] = {arm: [] for arm in ARMS} + for outcome in outcomes: + by_arm[outcome.arm_id].append(outcome) + + ar_outcomes = by_arm["ar_only"] + paired: dict[str, dict[str, int]] = {} + for arm in ARMS: + if arm == "ar_only": + continue + paired[arm] = paired_validity_table(ar_outcomes, by_arm[arm]) + + def valid_rate(arm: str) -> float: + rows = by_arm[arm] + return sum(o.v1_verdict for o in rows) / max(len(rows), 1) + + n_paired = sum(paired["ar_repair_improved"].values()) - paired["ar_repair_improved"]["unpaired"] + improvements = paired["ar_repair_improved"]["a_invalid_b_valid"] + damages = paired["ar_repair_improved"]["a_valid_b_invalid"] + wilson = wilson_interval(improvements, max(n_paired, 0)) + wilson_damage = wilson_interval(damages, max(n_paired, 0)) + + reachability = {p: Path(p).exists() for p in REACHABILITY_ARTIFACTS} + gates = { + "safety": { + "rule": PREREGISTERED["safety_rule"], + "invalid_over_valid": damages, + "pass": damages == 0, + }, + "value": { + "rule": PREREGISTERED["value_rule"], + "improvements": improvements, + "n_paired": n_paired, + "wilson": wilson, + "min_useful_effect": PREREGISTERED["min_useful_effect"], + "pass": bool( + wilson["low"] is not None + and wilson["low"] > PREREGISTERED["min_useful_effect"] + and damages == 0 + ), + }, + "reachability": { + "artifacts": reachability, + "pass": all(reachability.values()), + }, + } + + safety_pass = gates["safety"]["pass"] + value_pass = gates["value"]["pass"] + reach_pass = gates["reachability"]["pass"] + if safety_pass and value_pass and reach_pass: + disposition = DISPOSITION_POSITIVE + elif not safety_pass or ( + wilson["high"] is not None + and wilson["high"] < PREREGISTERED["min_useful_effect"] + ): + disposition = DISPOSITION_NEGATIVE + else: + disposition = DISPOSITION_INCONCLUSIVE + + lar3_statement = { + DISPOSITION_POSITIVE: ( + "LAR3 (learned repair integration) is OPEN: the do-no-harm hybrid " + "cleared safety, value, and reachability gates on the frozen screen." + ), + DISPOSITION_NEGATIVE: ( + "LAR3 (learned repair integration) is CLOSED: repair damaged valid " + "AR programs or its paired value was ruled out below the minimum " + "useful effect on the frozen screen." + ), + DISPOSITION_INCONCLUSIVE: ( + "LAR3 (learned repair integration) remains OPEN but NOT advanced: " + "the screen neither cleared the value gate nor ruled the effect " + "out at fixture power; a powered rerun is required before LAR3 " + "can open or close." + ), + }[disposition] + + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-317", + "question": ( + "Does a do-no-harm AR→repair hybrid (deterministic commit rule) " + "improve paired hard-valid outcomes over AR only without ever " + "damaging a valid AR program?" + ), + "preregistered": PREREGISTERED, + "arms": list(ARMS), + "promotable_arms": sorted(PROMOTABLE_ARMS), + "config": { + "n_train_decisions": n_train, + "n_eval_decisions": n_eval, + "seeds": list(seeds), + "steps": steps, + "lr": lr, + "scorer_steps": scorer_steps, + "fixture_config_overrides": FIXTURE_CONFIG_OVERRIDES, + "torch_num_threads": 1, + }, + "arm_summaries": { + arm: { + "n": len(by_arm[arm]), + "valid_rate": valid_rate(arm), + "mean_hard_rank": ( + sum(o.hard_rank for o in by_arm[arm]) / max(len(by_arm[arm]), 1) + ), + } + for arm in ARMS + }, + "paired_vs_ar_only": paired, + "invalid_over_valid_per_arm": { + arm: paired[arm]["a_valid_b_invalid"] for arm in paired + }, + "commit_reason_counts": { + arm: { + reason: sum(1 for o in by_arm[arm] if o.commit_reason == reason) + for reason in sorted( + {o.commit_reason for o in by_arm[arm] if o.commit_reason} + ) + } + for arm in ("ar_repair_historical", "ar_repair_improved", "oracle_commit") + }, + "gates": gates, + "wilson_damage": wilson_damage, + "oracle_upper_bound": { + "valid_rate": valid_rate("oracle_commit"), + "sanity": valid_rate("oracle_commit") >= valid_rate("ar_repair_improved"), + }, + "per_example_outcomes": [o.to_dict() for o in outcomes], + "decisions_path": str(decisions_out), + "disposition": disposition, + "lar3_statement": lar3_statement, + "wall_seconds": time.perf_counter() - start, + "honesty": ( + "Fixture-scale wiring screen: the frozen corpus is the SLM-155 " + "synthetic decision fixture (n eval decisions below any ship-gate " + "prompt count), models are tiny CPU fixtures trained for a handful " + "of steps, and no ship-gate claim is made. The commit rule is " + "deterministic and metamorphism-invariant by construction; the " + "screen measures the mechanism, not production quality. The " + "historical arm's pre-SLM-305 4-action space is declared " + "non-reproducible on this branch (see preregistered deviation)." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm317_repair_hybrid", + "harness.experiments.slm155_factorization_comparison", + "harness.experiments.slm299_edit_reachability", + ) + return payload + + +def render_markdown(payload: dict) -> str: + def f(x: float) -> str: + return f"{x:.3f}" + + lines = [ + "# SLM-317 (LAR2-06): do-no-harm AR→repair hybrid screen", + "", + f"**Disposition: `{payload['disposition']}`** — {payload['lar3_statement']}", + "", + "Fixture-scale mechanism screen; **not a ship claim**.", + "", + "## Preregistered (locked before results)", + "", + f"- safety: {payload['preregistered']['safety_rule']}", + f"- value: {payload['preregistered']['value_rule']}", + f"- disposition: {payload['preregistered']['disposition_rule']}", + f"- declared deviation: {payload['preregistered']['historical_deviation']}", + "", + "## Arms (matched examples + seeds + budgets)", + "", + "| arm | n | hard-valid rate | mean hard rank | invalid-over-valid vs AR |", + "| --- | --- | --- | --- | --- |", + ] + iov = payload["invalid_over_valid_per_arm"] + for arm in payload["arms"]: + s = payload["arm_summaries"][arm] + lines.append( + f"| {arm} | {s['n']} | {f(s['valid_rate'])} | " + f"{f(s['mean_hard_rank'])} | {iov.get(arm, '—')} |" + ) + lines += [ + "", + "## Paired outcomes vs `ar_only` (per-example, nothing aggregated away)", + "", + "| arm | both valid | AR valid, arm invalid | AR invalid, arm valid | both invalid | unpaired |", + "| --- | --- | --- | --- | --- | --- |", + ] + for arm, counts in payload["paired_vs_ar_only"].items(): + lines.append( + f"| {arm} | {counts['both_valid']} | {counts['a_valid_b_invalid']} | " + f"{counts['a_invalid_b_valid']} | {counts['both_invalid']} | {counts['unpaired']} |" + ) + gates = payload["gates"] + wilson = gates["value"]["wilson"] + lines += [ + "", + "## Advancement gates", + "", + f"- **Safety**: invalid-over-valid = {gates['safety']['invalid_over_valid']} " + f"(must be 0) → {'PASS' if gates['safety']['pass'] else 'FAIL'}", + f"- **Value**: improvements {gates['value']['improvements']}/{gates['value']['n_paired']}, " + f"Wilson 95% [{wilson['low']}, {wilson['high']}] vs minimum useful effect " + f"{gates['value']['min_useful_effect']} → {'PASS' if gates['value']['pass'] else 'FAIL'}", + f"- **Reachability/provenance**: slm299/291 artifacts present → " + f"{'PASS' if gates['reachability']['pass'] else 'FAIL'}", + "", + "## Commit reasons", + "", + "| arm | reason | count |", + "| --- | --- | --- |", + ] + for arm, reasons in payload["commit_reason_counts"].items(): + for reason, count in reasons.items(): + lines.append(f"| {arm} | {reason} | {count} |") + lines += [ + "", + f"Oracle commit upper bound hard-valid rate: " + f"{f(payload['oracle_upper_bound']['valid_rate'])} " + f"(sanity ≥ improved hybrid: {payload['oracle_upper_bound']['sanity']}).", + "", + f"Durable per-example commit decisions: `{payload['decisions_path']}`.", + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=6) + parser.add_argument("--lr", type=float, default=3e-3) + parser.add_argument("--n-train", type=int, default=16) + parser.add_argument("--n-eval", type=int, default=8) + parser.add_argument("--seeds", type=int, nargs="+", default=[0, 1]) + parser.add_argument("--scorer-steps", type=int, default=20) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + parser.add_argument("--decisions-out", type=Path, default=DEFAULT_DECISIONS_OUT) + args = parser.parse_args(argv) + + payload = run_screen( + n_train=args.n_train, + n_eval=args.n_eval, + seeds=tuple(args.seeds), + steps=args.steps, + lr=args.lr, + scorer_steps=args.scorer_steps, + decisions_out=args.decisions_out, + ) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"disposition={payload['disposition']} " + f"safety={payload['gates']['safety']['pass']} " + f"value={payload['gates']['value']['pass']} " + f"iov={payload['invalid_over_valid_per_arm'].get('ar_repair_improved')} " + f"-> {args.json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm317_repair_hybrid.py b/src/slm_training/harnesses/experiments/slm317_repair_hybrid.py new file mode 100644 index 000000000..2634f176d --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm317_repair_hybrid.py @@ -0,0 +1,446 @@ +"""SLM-317 (LAR2-06): do-no-harm AR→repair hybrid commit machinery. + +A valid-state repair arm is only useful if it never damages a correct AR +program. This module provides the deterministic, metamorphism-invariant +commit rule for the AR→repair hybrid screen: + +- :func:`hard_evidence` scores one final-program source on the LAR1-02 + unified ladder: official parse (``dsl.parser.validate``) → output contract + (``output_contract_violations``) → ``meaningful_program_v1`` verdict. The + ladder rank is the ONLY hard authority. +- :func:`commit` applies the preregistered do-no-harm rule: + + a. hard evidence improves → COMMIT (``hard_improvement``); + b. hard evidence regresses → RETAIN source (``hard_regression``); + c. hard evidence unchanged and a calibrated learned score (model value or + oracle distance improvement) improves → COMMIT + (``soft_improvement_no_hard_regression``); + d. otherwise RETAIN (``no_improvement``); a missing/empty candidate is an + ABSTAIN (``candidate_unavailable``), never a silent commit. + +- :func:`oracle_commit` is the non-promotable upper bound: commit exactly + when the candidate's hard evidence is strictly better. +- Metamorphic transforms (:func:`alpha_rename`, :func:`reorder_statements`, + :func:`normalize_formatting`, :func:`ast_roundtrip`) are + semantics-preserving source rewrites used by the invariance tests: hard + evidence and commit decisions must be identical across them. +- :func:`repair_decode` runs the branch's value-guided beam repair starting + from an arbitrary valid source state (not the minimal seed), fail-closed + via ``TreeEditSpace.apply``. + +The module is torch-free at import time; ``repair_decode``/``state_value`` +import torch lazily through the model they are given. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any + +from slm_training.dsl.language_contract import output_contract_violations +from slm_training.dsl.parser import validate +from slm_training.harnesses.model_build.eval_runner import ( + meaningful_program_v1_report, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_STOP, + Statement, + parse_statements, + render_statements, +) + +EXPERIMENT_ID = "slm317-repair-hybrid" +COMMIT_DECISION_SCHEMA = "slm317_commit_decision/v1" + +# Commit actions. +ACTION_COMMIT = "commit" +ACTION_RETAIN = "retain" +ACTION_ABSTAIN = "abstain" + +# Preregistered reason codes (the full closed set). +REASON_HARD_IMPROVEMENT = "hard_improvement" +REASON_HARD_REGRESSION = "hard_regression" +REASON_SOFT_IMPROVEMENT = "soft_improvement_no_hard_regression" +REASON_NO_IMPROVEMENT = "no_improvement" +REASON_CANDIDATE_UNAVAILABLE = "candidate_unavailable" +REASON_ORACLE_HARD_IMPROVEMENT = "oracle_hard_improvement" +REASON_ORACLE_NO_GAIN = "oracle_no_gain" + +# Dispositions (exactly one per screen run). +DISPOSITION_POSITIVE = "repair_positive" +DISPOSITION_NEGATIVE = "repair_negative" +DISPOSITION_INCONCLUSIVE = "inconclusive" + + +# --------------------------------------------------------------------------- # +# Hard evidence (LAR1-02 ladder) +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class HardEvidence: + """Deterministic verifier/contract evidence for one final program.""" + + parse_ok: bool + contract_ok: bool + v1_verdict: bool + v1_reason_codes: tuple[str, ...] + contract_violations: tuple[str, ...] + + @property + def rank(self) -> int: + """Ordered ladder: 0 unparseable < 1 parsed < 2 contract-clean < 3 v1-valid.""" + if not self.parse_ok: + return 0 + if not self.contract_ok: + return 1 + if not self.v1_verdict: + return 2 + return 3 + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["v1_reason_codes"] = list(self.v1_reason_codes) + data["contract_violations"] = list(self.contract_violations) + data["rank"] = self.rank + return data + + +def hard_evidence(source: str | None) -> HardEvidence: + """Score one program source on the deterministic ladder. Never raises.""" + if not source: + return HardEvidence(False, False, False, ("no_program",), ()) + parse_ok = False + try: + validate(source) + parse_ok = True + except Exception: # noqa: BLE001 - parse fact, never raised + parse_ok = False + violations: tuple[str, ...] = () + if parse_ok: + violations = tuple(output_contract_violations(source)) + contract_ok = parse_ok and not violations + report = meaningful_program_v1_report(source, gold=None) + return HardEvidence( + parse_ok=parse_ok, + contract_ok=contract_ok, + v1_verdict=bool(report["verdict"]), + v1_reason_codes=tuple(report.get("reason_codes", ())), + contract_violations=violations, + ) + + +# --------------------------------------------------------------------------- # +# Commit rule +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class CommitDecision: + """Durable per-example commit record (source + candidate + exact reason).""" + + action: str # commit | retain | abstain + reason: str + source: str + candidate: str + final: str + source_evidence: HardEvidence + candidate_evidence: HardEvidence + learned_score_source: float | None = None + learned_score_candidate: float | None = None + record_id: str = "" + arm_id: str = "" + seed: int = 0 + notes: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": COMMIT_DECISION_SCHEMA, + "action": self.action, + "reason": self.reason, + "source": self.source, + "candidate": self.candidate, + "final": self.final, + "source_evidence": self.source_evidence.to_dict(), + "candidate_evidence": self.candidate_evidence.to_dict(), + "learned_score_source": self.learned_score_source, + "learned_score_candidate": self.learned_score_candidate, + "record_id": self.record_id, + "arm_id": self.arm_id, + "seed": self.seed, + "notes": list(self.notes), + } + + +def commit( + source: str, + candidate: str | None, + *, + evidence: dict[str, Any] | None = None, + record_id: str = "", + arm_id: str = "", + seed: int = 0, +) -> CommitDecision: + """Do-no-harm commit rule (deterministic; see module docstring). + + ``evidence`` may carry ``learned_score_source`` / + ``learned_score_candidate`` (model value or oracle distance improvement; + higher = better) and ``learned_margin`` (strict improvement margin, + default 0.0). Learned scores are only consulted when hard evidence is + unchanged, and a soft commit is forbidden whenever hard evidence regresses. + """ + evidence = evidence or {} + src_ev = hard_evidence(source) + if candidate is None or not candidate.strip(): + return CommitDecision( + action=ACTION_ABSTAIN, + reason=REASON_CANDIDATE_UNAVAILABLE, + source=source, + candidate="", + final=source, + source_evidence=src_ev, + candidate_evidence=hard_evidence(None), + record_id=record_id, + arm_id=arm_id, + seed=seed, + ) + cand_ev = hard_evidence(candidate) + soft_s = evidence.get("learned_score_source") + soft_c = evidence.get("learned_score_candidate") + margin = float(evidence.get("learned_margin", 0.0)) + + if cand_ev.rank > src_ev.rank: + action, reason, final = ACTION_COMMIT, REASON_HARD_IMPROVEMENT, candidate + elif cand_ev.rank < src_ev.rank: + action, reason, final = ACTION_RETAIN, REASON_HARD_REGRESSION, source + elif ( + soft_s is not None + and soft_c is not None + and float(soft_c) > float(soft_s) + margin + ): + action, reason, final = ACTION_COMMIT, REASON_SOFT_IMPROVEMENT, candidate + else: + action, reason, final = ACTION_RETAIN, REASON_NO_IMPROVEMENT, source + return CommitDecision( + action=action, + reason=reason, + source=source, + candidate=candidate, + final=final, + source_evidence=src_ev, + candidate_evidence=cand_ev, + learned_score_source=None if soft_s is None else float(soft_s), + learned_score_candidate=None if soft_c is None else float(soft_c), + record_id=record_id, + arm_id=arm_id, + seed=seed, + ) + + +def oracle_commit( + source: str, + candidate: str | None, + *, + record_id: str = "", + arm_id: str = "", + seed: int = 0, +) -> CommitDecision: + """Non-promotable upper bound: commit exactly on strict hard improvement.""" + if candidate is None or not candidate.strip(): + return commit( + source, candidate, record_id=record_id, arm_id=arm_id, seed=seed + ) + src_ev = hard_evidence(source) + cand_ev = hard_evidence(candidate) + better = cand_ev.rank > src_ev.rank + return CommitDecision( + action=ACTION_COMMIT if better else ACTION_RETAIN, + reason=REASON_ORACLE_HARD_IMPROVEMENT if better else REASON_ORACLE_NO_GAIN, + source=source, + candidate=candidate, + final=candidate if better else source, + source_evidence=src_ev, + candidate_evidence=cand_ev, + record_id=record_id, + arm_id=arm_id, + seed=seed, + notes=["oracle selector; non-promotable upper bound"], + ) + + +# --------------------------------------------------------------------------- # +# Metamorphic transforms (semantics-preserving; used by invariance tests) +# --------------------------------------------------------------------------- # + + +def alpha_rename(source: str, *, prefix: str = "m") -> str: + """Consistently rename non-root node identifiers (references included).""" + statements = parse_statements(source) + if statements is None: + return source + mapping: dict[str, str] = {} + for stmt in statements: + if stmt.name != "root": + mapping[stmt.name] = f"{prefix}{len(mapping)}" + renamed = [ + Statement( + name=mapping.get(stmt.name, stmt.name), + comp=stmt.comp, + children=[mapping.get(c, c) for c in stmt.children], + rest=stmt.rest, + has_list=stmt.has_list, + ) + for stmt in statements + ] + return render_statements(renamed) + + +def reorder_statements(source: str) -> str: + """Move the root statement last (declarations first); semantics preserved.""" + statements = parse_statements(source) + if statements is None: + return source + roots = [s for s in statements if s.name == "root"] + others = [s for s in statements if s.name != "root"] + return render_statements(others + roots) + + +def normalize_formatting(source: str) -> str: + """Re-render the structural form (canonical spacing/newlines).""" + statements = parse_statements(source) + if statements is None: + return source.strip() + return render_statements(statements) + + +def ast_roundtrip(source: str) -> str: + """Equivalent AST serialization via the official parser's serialized form.""" + try: + program = validate(source) + except Exception: # noqa: BLE001 - unparseable input round-trips to itself + return source + return program.serialized or source.strip() + + +# --------------------------------------------------------------------------- # +# Repair decode from an arbitrary valid source state +# --------------------------------------------------------------------------- # + + +def repair_decode( + model: Any, + source: str, + inventory: list[str], + prompt: str, +) -> tuple[str, dict[str, Any]]: + """Value-guided beam repair seeded from ``source`` (not the minimal seed). + + Mirrors ``TreeEditDiffusionModel._decode_one`` (same fail-closed + ``TreeEditSpace.apply``, same value re-scoring, same STOP handling) with + the beam initialized on the caller's valid state. An unparseable or + invalid source fails closed: returns ``("", {"failure": ...})``. + """ + import torch # lazy: module import stays torch-free + + statements = parse_statements(source) + if statements is None: + return "", {"failure": "source_unparseable", "kind": "tree_edit_repair"} + from slm_training.models.tree_edit_diffusion import _is_valid + + if not _is_valid(source): + return "", {"failure": "source_invalid", "kind": "tree_edit_repair"} + + model.eval() + ctx, ctx_pad = model._encode_context( + [model._format_context(prompt, slot_contract=list(inventory))] + ) + cfg = model.config + beam: list[tuple[float, list[Statement], bool]] = [(0.0, statements, False)] + evidence: dict[str, Any] = {"steps": 0, "expansions": 0, "kind": "tree_edit_repair"} + with torch.no_grad(): + for _ in range(cfg.max_search_steps): + live = [entry for entry in beam if not entry[2]] + if not live: + break + sources = [render_statements(s) for _, s, _ in live] + out = model.policy( + model._state_batch(sources), + model.tokenizer.pad_id, + ctx.expand(len(sources), -1, -1), + ctx_pad.expand(len(sources), -1), + ) + next_beam = [entry for entry in beam if entry[2]] + seen: set[str] = { + render_statements(s) for _, s, frozen in next_beam if frozen + } + for row, (_, stmts, _) in enumerate(live): + candidates = model._enumerate_edits(out, row, len(stmts), len(inventory)) + expanded = 0 + for score, edit in candidates: + if expanded >= cfg.expand_per_state: + break + if edit.action == ACTION_STOP: + text = render_statements(stmts) + if text in seen: + continue + seen.add(text) + next_beam.append((float(out["value"][row]), stmts, True)) + expanded += 1 + continue + child = model.space.apply(stmts, edit, inventory, reason=[]) + if child is None: + continue + text = render_statements(child) + if text in seen: + continue + seen.add(text) + next_beam.append((float(out["value"][row]), child, False)) + expanded += 1 + evidence["expansions"] += 1 + if not next_beam: + break + unfrozen = [entry for entry in next_beam if not entry[2]] + if unfrozen: + rescore = model.policy( + model._state_batch([render_statements(s) for _, s, _ in unfrozen]), + model.tokenizer.pad_id, + ctx.expand(len(unfrozen), -1, -1), + ctx_pad.expand(len(unfrozen), -1), + ) + rescored = [ + (float(rescore["value"][i]), entry[1], False) + for i, entry in enumerate(unfrozen) + ] + else: + rescored = [] + frozen = [entry for entry in next_beam if entry[2]] + beam = sorted(frozen + rescored, key=lambda e: e[0], reverse=True)[ + : cfg.beam_width + ] + evidence["steps"] += 1 + if all(entry[2] for entry in beam): + break + best = max(beam, key=lambda e: e[0]) + evidence["value"] = float(best[0]) + evidence["frozen"] = bool(best[2]) + return render_statements(best[1]), evidence + + +def state_value(model: Any, source: str, inventory: list[str], prompt: str) -> float | None: + """Calibrated learned score (policy value head) for one state; None if unparseable.""" + import torch # lazy + + if parse_statements(source) is None: + return None + model.eval() + ctx, ctx_pad = model._encode_context( + [model._format_context(prompt, slot_contract=list(inventory))] + ) + with torch.no_grad(): + out = model.policy( + model._state_batch([source]), + model.tokenizer.pad_id, + ctx, + ctx_pad, + ) + return float(out["value"][0]) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 0edc43962..dadbf251a 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -7879,6 +7879,24 @@ "note": "initial registration; SLM-314 LAR2-05 winner-take-all (MCL): frozen 8-prompt x 2-mode verifier-accepted canonical-AST dataset (alpha-invariant fingerprints + sha256 manifest), min-loss-mode loss with preregistered 0.1 coverage/floor term, deterministic two-mode synthetic collapse-vs-retain proof, matched {single_gold, multi_gold, wta} arms" } ] + }, + "harness.experiments.slm317_repair_hybrid": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm317_repair_hybrid.py", + "scripts/run_slm317_repair_hybrid.py", + "tests/test_harnesses/experiments/test_slm317_repair_hybrid.py", + "docs/design/iter-slm317-repair-hybrid-20260724.json", + "docs/design/iter-slm317-repair-hybrid-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-317 LAR2-06 do-no-harm AR->repair hybrid: metamorphism-invariant commit rule (hard ladder + calibrated soft score), matched arms incl. historical/improved repair + oracle upper bound, invalid-over-valid visibility, advancement gates and repair_positive|repair_negative|inconclusive disposition" + } + ] } } } diff --git a/tests/test_harnesses/experiments/test_slm317_repair_hybrid.py b/tests/test_harnesses/experiments/test_slm317_repair_hybrid.py new file mode 100644 index 000000000..76c7a93bf --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm317_repair_hybrid.py @@ -0,0 +1,263 @@ +"""SLM-317 (LAR2-06): metamorphic invariance, commit truth table, visibility. + +Runs before any model evaluation: the commit/score pipeline must be invariant +to alpha-renaming, semantics-preserving reordering, formatting normalization, +and equivalent AST serializations — and the do-no-harm rule must cover its +full truth table deterministically. +""" + +from __future__ import annotations + +import json + +import pytest + +from slm_training.harnesses.experiments.slm155_factorization_comparison import ( + paired_validity_table, +) +from slm_training.harnesses.experiments.slm317_repair_hybrid import ( + ACTION_ABSTAIN, + ACTION_COMMIT, + ACTION_RETAIN, + REASON_CANDIDATE_UNAVAILABLE, + REASON_HARD_IMPROVEMENT, + REASON_HARD_REGRESSION, + REASON_NO_IMPROVEMENT, + REASON_ORACLE_NO_GAIN, + REASON_SOFT_IMPROVEMENT, + alpha_rename, + ast_roundtrip, + commit, + hard_evidence, + normalize_formatting, + oracle_commit, + reorder_statements, +) +from scripts.run_slm317_repair_hybrid import ArmOutcome + +VALID = 'root = Stack([n0], "column")\nn0 = TextContent(":content")' +VALID_TWO = ( + 'root = Stack([n0, n1], "column")\n' + 'n0 = TextContent(":content")\n' + 'n1 = Button(":cta")' +) +INVALID_UNPARSEABLE = "root = Stack(([n0]," +VALID_PARSE_ONLY = 'root = Stack([], "column")' # parses; v1 invalid (empty stack) + +TRANSFORMS = (alpha_rename, reorder_statements, normalize_formatting, ast_roundtrip) + + +# --------------------------------------------------------------------------- # +# Metamorphic invariance (all four transforms) +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("source", [VALID, VALID_TWO, VALID_PARSE_ONLY]) +def test_hard_evidence_invariant_across_transforms(source: str) -> None: + base = hard_evidence(source) + for transform in TRANSFORMS: + variant = transform(source) + ev = hard_evidence(variant) + assert ev.rank == base.rank, transform.__name__ + assert ev.parse_ok == base.parse_ok + assert ev.contract_ok == base.contract_ok + assert ev.v1_verdict == base.v1_verdict + + +def test_commit_decision_invariant_across_transforms() -> None: + evidence = {"learned_score_source": 0.2, "learned_score_candidate": 0.9} + base = commit(VALID, VALID_TWO, evidence=evidence) + for transform in TRANSFORMS: + variant = commit(transform(VALID), transform(VALID_TWO), evidence=evidence) + assert variant.action == base.action, transform.__name__ + assert variant.reason == base.reason, transform.__name__ + assert variant.source_evidence.rank == base.source_evidence.rank + assert variant.candidate_evidence.rank == base.candidate_evidence.rank + + +def test_transforms_preserve_validity() -> None: + for transform in TRANSFORMS: + assert hard_evidence(transform(VALID)).v1_verdict, transform.__name__ + + +# --------------------------------------------------------------------------- # +# Commit-rule truth table (all branches + determinism) +# --------------------------------------------------------------------------- # + + +def test_truth_table_hard_improvement() -> None: + decision = commit(VALID_PARSE_ONLY, VALID) + assert decision.action == ACTION_COMMIT + assert decision.reason == REASON_HARD_IMPROVEMENT + assert decision.final == VALID + + +def test_truth_table_hard_regression_retains_source() -> None: + decision = commit(VALID, INVALID_UNPARSEABLE) + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_HARD_REGRESSION + assert decision.final == VALID + + +def test_truth_table_soft_improvement_no_hard_regression() -> None: + candidate = alpha_rename(VALID) # same hard rank, different source + decision = commit( + VALID, + candidate, + evidence={"learned_score_source": 0.1, "learned_score_candidate": 0.9}, + ) + assert decision.action == ACTION_COMMIT + assert decision.reason == REASON_SOFT_IMPROVEMENT + assert decision.final == candidate + + +def test_soft_commit_forbidden_when_hard_regresses() -> None: + decision = commit( + VALID, + INVALID_UNPARSEABLE, + evidence={"learned_score_source": 0.1, "learned_score_candidate": 0.9}, + ) + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_HARD_REGRESSION + + +def test_truth_table_no_improvement() -> None: + candidate = alpha_rename(VALID) + decision = commit(VALID, candidate) # no learned scores, same rank + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_NO_IMPROVEMENT + assert decision.final == VALID + + +def test_soft_margin_not_exceeded_retains() -> None: + candidate = alpha_rename(VALID) + decision = commit( + VALID, + candidate, + evidence={ + "learned_score_source": 0.5, + "learned_score_candidate": 0.5, + "learned_margin": 0.0, + }, + ) + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_NO_IMPROVEMENT + + +def test_truth_table_abstain_on_missing_candidate() -> None: + for candidate in (None, "", " "): + decision = commit(VALID, candidate) + assert decision.action == ACTION_ABSTAIN + assert decision.reason == REASON_CANDIDATE_UNAVAILABLE + assert decision.final == VALID + + +def test_commit_deterministic() -> None: + evidence = {"learned_score_source": 0.1, "learned_score_candidate": 0.9} + first = commit(VALID, VALID_TWO, evidence=evidence).to_dict() + second = commit(VALID, VALID_TWO, evidence=evidence).to_dict() + assert first == second + + +def test_decision_preserves_source_candidate_reason() -> None: + decision = commit(VALID_PARSE_ONLY, VALID, record_id="r1", arm_id="a", seed=3) + row = decision.to_dict() + assert row["source"] == VALID_PARSE_ONLY + assert row["candidate"] == VALID + assert row["reason"] == REASON_HARD_IMPROVEMENT + assert row["record_id"] == "r1" + assert row["schema"] == "slm317_commit_decision/v1" + json.dumps(row) # durable JSONL-serializable + + +# --------------------------------------------------------------------------- # +# Oracle selector upper bound +# --------------------------------------------------------------------------- # + + +def test_oracle_commits_only_on_strict_hard_improvement() -> None: + better = oracle_commit(VALID_PARSE_ONLY, VALID) + assert better.action == ACTION_COMMIT + worse = oracle_commit(VALID, VALID_PARSE_ONLY) + assert worse.action == ACTION_RETAIN + assert worse.reason == REASON_ORACLE_NO_GAIN + equal = oracle_commit(VALID, alpha_rename(VALID)) + assert equal.action == ACTION_RETAIN # equal rank is not an improvement + + +def test_oracle_upper_bound_dominates_rule() -> None: + # Whatever the deterministic rule commits, the oracle commits a superset + # (it commits on every strict hard improvement; the rule adds soft commits + # only when hard evidence is unchanged, which can never beat the oracle's + # final rank). + cases = [ + (VALID_PARSE_ONLY, VALID), + (VALID, VALID_PARSE_ONLY), + (VALID, INVALID_UNPARSEABLE), + (VALID, alpha_rename(VALID)), + ] + for source, candidate in cases: + rule = commit( + source, + candidate, + evidence={"learned_score_source": 0.0, "learned_score_candidate": 1.0}, + ) + oracle = oracle_commit(source, candidate) + assert oracle.source_evidence.rank <= oracle.candidate_evidence.rank or ( + oracle.action == ACTION_RETAIN + ) + assert hard_evidence(oracle.final).rank >= hard_evidence(rule.final).rank + + +# --------------------------------------------------------------------------- # +# Invalid-over-valid visibility + arm isolation +# --------------------------------------------------------------------------- # + + +def _outcome(record_id: str, arm: str, seed: int, verdict: bool) -> ArmOutcome: + return ArmOutcome( + record_id=record_id, + arm_id=arm, + seed=seed, + final_source=VALID if verdict else INVALID_UNPARSEABLE, + v1_verdict=verdict, + hard_rank=3 if verdict else 0, + ) + + +def test_invalid_over_valid_damage_surfaces() -> None: + """AR final valid + repaired final invalid must be visible, never hidden.""" + ar = [_outcome("r0", "ar_only", 0, True), _outcome("r1", "ar_only", 0, True)] + repaired = [ + _outcome("r0", "ar_repair_improved", 0, True), + _outcome("r1", "ar_repair_improved", 0, False), # damaged + ] + counts = paired_validity_table(ar, repaired) + assert counts["a_valid_b_invalid"] == 1 # the damage count is explicit + assert counts["both_valid"] == 1 + + +def test_commit_rule_makes_damage_unreachable() -> None: + """With the commit rule in the loop, a damaging candidate is retained away.""" + decision = commit(VALID, INVALID_UNPARSEABLE) + final_ev = hard_evidence(decision.final) + assert final_ev.v1_verdict # final is the retained valid AR source + + +def test_arm_isolation() -> None: + """Outcomes from one arm never leak into another arm's pairing.""" + ar = [_outcome("r0", "ar_only", 0, True)] + other = [_outcome("r0", "repair_only", 0, False)] + counts = paired_validity_table(ar, other) + assert counts["a_valid_b_invalid"] == 1 + # A third arm with no matching record contributes only unpaired rows. + third = [_outcome("rX", "ar_repair_improved", 0, True)] + counts2 = paired_validity_table(ar, third) + assert counts2["unpaired"] == 2 + assert counts2["a_valid_b_invalid"] == 0 + + +def test_paired_table_deterministic() -> None: + ar = [_outcome(f"r{i}", "ar_only", 0, i % 2 == 0) for i in range(4)] + arm = [_outcome(f"r{i}", "ar_repair_improved", 0, True) for i in range(4)] + assert paired_validity_table(ar, arm) == paired_validity_table(ar, arm) From e258c9173c2da0637ae41ed2c4405183f9e610b2 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:28:28 -0500 Subject: [PATCH 10/10] =?UTF-8?q?SLM-319:=20close=20LAR3-01=20not=5Fauthor?= =?UTF-8?q?ized=20=E2=80=94=20entry=20gates=20unmet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LAR0-02 returned recursive_core_negative (1 of 2 required seed passes) and LAR2-06 returned inconclusive with the value gate failed (0/16, Wilson [0.0, 0.194] vs 0.05); LAR3 is open but NOT advanced. No production code, defaults, or checkpoints; LAR3-02..05 stay blocked. Reopening requires recursive_core_positive plus a passing repair advancement screen. --- ...er-slm319-lar3-gate-closeout-20260725.json | 49 +++++++++++++++++++ ...iter-slm319-lar3-gate-closeout-20260725.md | 33 +++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 docs/design/iter-slm319-lar3-gate-closeout-20260725.json create mode 100644 docs/design/iter-slm319-lar3-gate-closeout-20260725.md diff --git a/docs/design/iter-slm319-lar3-gate-closeout-20260725.json b/docs/design/iter-slm319-lar3-gate-closeout-20260725.json new file mode 100644 index 000000000..8dfa80de0 --- /dev/null +++ b/docs/design/iter-slm319-lar3-gate-closeout-20260725.json @@ -0,0 +1,49 @@ +{ + "claim_class": "wiring", + "decision": "not_authorized", + "downstream_effect": "SLM-321/324/326/327 (LAR3-02..05) stay blocked on SLM-319. Reopen only when a recurrence-health audit returns recursive_core_positive AND a valid-state repair advancement screen passes its value gate.", + "failed_gates": [ + "gate_lar0_02", + "gate_lar2_06" + ], + "gates": { + "gate_lar0_01": { + "issue": "SLM-279", + "observed_outcome": "Done (correction-only); neutral for entry", + "passed": true, + "required_outcome": "canonical depth-supervision objective defined" + }, + "gate_lar0_02": { + "evidence": "docs/design/iter-slm282-recurrence-health-20260723.json", + "issue": "SLM-282", + "observed_outcome": "recursive_core_negative (1 of 2 required seed passes)", + "passed": false, + "required_outcome": "recursive_core_positive (>=2 seed passes)" + }, + "gate_lar2_06": { + "evidence": "docs/design/iter-slm317-repair-hybrid-20260724.json", + "issue": "SLM-317", + "observed_outcome": "inconclusive; safety PASS, value FAIL (0/16, Wilson [0.0, 0.194] vs 0.05); LAR3 open but NOT advanced", + "passed": false, + "required_outcome": "advancement screen passes safety + value gates" + } + }, + "issue": "SLM-319", + "matrix_set": "slm319-prelude-coda-core", + "matrix_version": "slm319-v1", + "production_code_added": false, + "production_default_changed": false, + "reason": "LAR3 entry prerequisites are not met: SLM-282 (LAR0-02) returned recursive_core_negative (1 of 2 required seed passes on the preregistered recurrence-health condition), and SLM-317 (LAR2-06) returned inconclusive with LAR3 explicitly 'open but NOT advanced' after the value gate failed (0/16 improvements, Wilson [0.0, 0.194] vs 0.05). Per the program's own advancement rules, the untied prelude/shared-core/coda architecture must not be implemented.", + "run_id": "slm319_gate_closeout", + "schema": "slm319_lar3_gate_closeout/v1", + "status": "closeout", + "version_stamp": { + "code_commit": "48e5cadc13f9e6e431491da615badb4a0614c17a", + "code_dirty": false, + "components": { + "harness.experiments": "v83" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:27:51.049596+00:00" + } +} diff --git a/docs/design/iter-slm319-lar3-gate-closeout-20260725.md b/docs/design/iter-slm319-lar3-gate-closeout-20260725.md new file mode 100644 index 000000000..c93ff7983 --- /dev/null +++ b/docs/design/iter-slm319-lar3-gate-closeout-20260725.md @@ -0,0 +1,33 @@ +# SLM-319 / LAR3-01: LAR3 entry-gate closeout (slm319_gate_closeout) + +Matrix set: `slm319-prelude-coda-core` · Version: `slm319-v1` · Status: **closeout** +Decision: **not_authorized** — no production code. + +## Gate assessment + +The issue is blocked by LAR0-01, LAR0-02, and LAR2-06; its own advancement +rules ("Only `recursive_core_positive` can satisfy a prerequisite for LAR3"; +"Compare only after LAR3 entry gates pass") require all of them. They do not +pass. + +| Gate | Issue | Required | Observed | Passed | +| --- | --- | --- | --- | --- | +| LAR0-01 | SLM-279 | canonical depth-supervision objective | Done (correction-only); neutral | True | +| LAR0-02 | SLM-282 | `recursive_core_positive` (≥2 seed passes) | `recursive_core_negative` — 1 of 2 required seed passes ([json](iter-slm282-recurrence-health-20260723.json)) | **False** | +| LAR2-06 | SLM-317 | advancement screen passes safety + value gates | `inconclusive` — safety PASS, value FAIL (0/16, Wilson [0.0, 0.194] vs 0.05); LAR3 open but NOT advanced ([json](iter-slm317-repair-hybrid-20260724.json)) | **False** | + +## Consequences + +- No prelude/shared-core/coda architecture is implemented; no config, + checkpoint, or default change. +- SLM-321/324/326/327 (LAR3-02..05) stay blocked on this issue. + +## Reopening conditions + +Reopen or supersede only when **both**: + +1. a recurrence-health audit returns `recursive_core_positive` (≥2 seeds on + the preregistered condition), and +2. a valid-state repair advancement screen passes its preregistered value + gate (paired semantic improvement with Wilson lower bound above the + minimum useful effect).