From dac9a9ee1d2a12021b514437eda89523dda92ac7 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:23:34 -0500 Subject: [PATCH 01/18] SLM-301: prompt-contract observability with matched slot-inventory arms - coverage_class per frozen row (OBSERVABLE_PROMPT / OBSERVABLE_REQUEST_ONLY / UNKNOWN); all 51 rows are OBSERVABLE_REQUEST_ONLY pre-inventory. - Matched requests differ only by the production ensure_prompt_inventory suffix; request-derived, no hidden gold channel. - X22 deterministic arm: paired v2-strict delta +0.3125 (5/16 vs 0/16, Wilson [0.142, 0.556]) exceeds the predeclared 0.10 minimum; no regressions on observable rows. - AR tiny baseline: 6/32 decodes completed (all v2-fail partials), 26 decode_timeout (600-step checkpoint constrained-forest cost per SLM-294 evidence); reported unmeasured, not evidence. Rico rows not_run with SLM-294 cost evidence. - 100-record blind slot-observability audit: raw agreement 97%, kappa 0.0 (skew caveat); finding: v2 coverage detector misses inline slot enumeration -> false prompt_contract_unknown (append-only; no metric change). - 8 harness tests; new component harness.experiments.slm301_prompt_observability v1. --- src/slm_training/resources/versions.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 31b0b7bbf..40e8c67b4 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -4502,6 +4502,25 @@ } ] }, + "harness.experiments.slm301_prompt_observability": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm301_prompt_observability.py", + "scripts/run_slm301_prompt_observability.py", + "tests/test_harnesses/experiments/test_slm301_prompt_observability.py", + "docs/design/iter-slm301-prompt-observability-20260724.json", + "docs/design/iter-slm301-prompt-observability-20260724.md", + "docs/design/slm301-slot-observability-audit-20260724.json" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-301 LAR1-04 prompt-observability arms (as-is vs +inventory) with phase-0 coverage classification, AR/X22 baselines, paired predeclared-delta report" + } + ] + }, "harness.model_build.eval": { "version": "v52", "kind": "harness", From 8c1c2d9482689d06d249dff4f132a75247a71dc1 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:25:02 -0500 Subject: [PATCH 02/18] SLM-303: decode-budget/harness-interference audit on retained checkpoints - decode_outcome.py taxonomy with strict precedence (fallback never counts as model success); per-request budget/elapsed/forwards/ verifier/fallback/stop-reason records. - eval_runner additive integration: details[] gain decode_outcome, stop_reason, fallback_used; suite metrics gain decode_outcome_counts. - Census: only slm230 committed-SHA verifiable; hash-pinned the two remediated nonzero-timeout checkpoints; scoreboard classes separated (runtime_timeout/fallback/unmeasured/model_behavior). - Preregistered 1x vs 10x budget sweep: all 12 cells not_rerunnable (v0 checkpoints fail current output-contract check); the 10x-flip question is honestly UNANSWERED by re-decode. Recommendation: retrain the remediated recipe on contract v2, then re-eval at both budgets. - 25 new tests; harness.model_build.eval v50; component harness.experiments.slm303_decode_budget_audit v1; gates.ship v3 threshold mirror fixed. --- ...r-slm303-decode-budget-audit-20260724.json | 24024 ++++++++++++++++ ...ter-slm303-decode-budget-audit-20260724.md | 54 + scripts/run_slm303_decode_budget_audit.py | 917 + .../harnesses/model_build/decode_outcome.py | 140 + .../harnesses/model_build/eval_runner.py | 93 +- .../harnesses/model_build/ship_gates.py | 2 +- src/slm_training/resources/versions.json | 25 +- .../model_build/test_decode_outcome.py | 115 + .../test_run_slm303_decode_budget_audit.py | 159 + 9 files changed, 25524 insertions(+), 5 deletions(-) create mode 100644 docs/design/iter-slm303-decode-budget-audit-20260724.json create mode 100644 docs/design/iter-slm303-decode-budget-audit-20260724.md create mode 100644 scripts/run_slm303_decode_budget_audit.py create mode 100644 src/slm_training/harnesses/model_build/decode_outcome.py create mode 100644 tests/test_harnesses/model_build/test_decode_outcome.py create mode 100644 tests/test_scripts/test_run_slm303_decode_budget_audit.py diff --git a/docs/design/iter-slm303-decode-budget-audit-20260724.json b/docs/design/iter-slm303-decode-budget-audit-20260724.json new file mode 100644 index 000000000..42b2624fa --- /dev/null +++ b/docs/design/iter-slm303-decode-budget-audit-20260724.json @@ -0,0 +1,24024 @@ +{ + "schema": "slm303_decode_budget_audit/v1", + "generated_at": "2026-07-24T19:08:59Z", + "predeclaration": { + "experiment_id": "SLM-303", + "hypothesis": "A material share of historical zero/interference-flagged scoreboards are runtime/harness budget artifacts (decode timeout or fallback), not model behavior; widening the decode budget 10x flips timeout-classified rows to model outcomes.", + "falsifier": "10x budget does not change any timeout-classified record's outcome on the hash-pinned checkpoints, or the census shows the flagged scoreboards were already clean model behavior.", + "arms": { + "baseline": "recorded canvas cap (256/128) + inferred 10s timeout", + "budget10x": "10x timeout and 10x canvas cap; nothing else varies" + }, + "budget_evidence": "inferred from recorded eval_smoke.json: all 3 smoke rows share latency_ms=3333.79 (one chunk of 3 killed at ~10001ms \u2192 10s timeout); the original CLI flag was not persisted (config=null), disclosed as inferred", + "records": { + "ids": [ + "smoke_hero_01", + "smoke_button_01", + "smoke_callout_01" + ], + "source": "src/slm_training/resources/test_seeds.jsonl", + "disclosure": "the original eval suite directory is empty; the sweep rebuilds the same 3 smoke records (ids match the recorded scoreboard details) from test_seeds.jsonl" + }, + "seed": 0, + "fixed": "same checkpoint bytes (hash-pinned), same records, same seed, checkpoint-declared decode policy; arms differ only in budget", + "checkpoints": { + "ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "recorded_canvas_cap": 256, + "recorded_decode_timeout_count": 3, + "selection": "only checkpoints with ACTUAL nonzero-timeout scoreboards" + }, + "lexer_ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "recorded_canvas_cap": 128, + "recorded_decode_timeout_count": 3, + "selection": "only checkpoints with ACTUAL nonzero-timeout scoreboards" + } + }, + "excluded": { + "slm230_bounded_recursive_r4_r2": "zero-timeout scoreboard; budget interference not implicated \u2014 not needed", + "e173/slm294 and all other local checkpoints": "no committed SHA and no hash-pinned timeout scoreboard \u2192 not hash-verifiable for this audit (slm294 additionally feasibility-excluded: >280s/16 tokens constrained decode on rico)" + }, + "claim_class": "diagnostic" + }, + "census": { + "schema": "slm303_decode_census/v1", + "generated_at": "2026-07-24T19:08:59Z", + "taxonomy": [ + "model_valid", + "model_invalid", + "model_abstain", + "runtime_timeout", + "fallback_output", + "harness_error" + ], + "checkpoints": [ + { + "path": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e173-schema-context-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "5a84337262a851cfb74c7fed0e8c5d4f49112c75f7a82f4d3c0030d6f5c27f7f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "15a95b415884f59ffafea3e69826a41534b15afb72aeccecb85cd2099108f3b9", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "ed96323a4673c1ef4b6aa9c2c281e0b28a67c05ca0c7ee245ef8b06524ec87bc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e175-retrieval-8step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "ef8497ea80cd413e0c0e47f2d317ec6a331a8353ea94a11a16fd2feb682f0a34", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e175-retrieval-8step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "4e5b59d4abee56d3c155d9fba9c9d1503207f7dc275bea7aabde32b27023f3df", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e176-broad-corpus-8step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "996a5a43220b8c0216d389164b9056d8817e1fa9523ed88ff0560f6cee6c4ed1", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e176-broad-corpus-8step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f053e1604038c30afde727a3e2f73637ddb28624eeab7268e2f8ed0ac6188aa5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e177-semantic-judge-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "afd17c1ec60b0cc8540c09a4c66896c1b8a69ff44acf99c7ad9e754f1cbae81b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1ef4a2bfe6fb545e0bbb32ad811b515538dbc60ff13b64d3576875545bdedcf8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "0b6bb40e9968eb659ce668c34ecd7ece414eb3cf6d59e7769807e4d3e51269f3", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "453208f977d95b1c97d1c776558ddfa8fae000151fbf99a4ef3440f443deeda0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "22b8bd63e06670065e06b70f574c0a0123a831ec7324346c6041138f3010dcac", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e195-stratified-compiler-aligned-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "f8eb684b48dc34b90223fb8ecadf1210e5b0da702d3611b18e0f18d63e5b127e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e195-stratified-compiler-aligned-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "5db0f544a1bae7f9bac0386fd997e0ab53f877b8efc25df65083f6856952b5a7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c4308ed8ad6765a6fe44c0c21ffd9c5feb5b4c03ba28e0ebcb21a5fc0d86f01d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "53f14d5b8470cbe25e7fb1f97982e1978ecd57035149eb8a747f5a79d8106f30", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9259d61f2d30c1b08ad37f1266778a203efdfbe154de602a58616f68d069565e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "3bf8b8d25309818e438fd2814796edf3bd5825b454fd900ffdb80f2989615e4f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e8e0ee6ef5b79d930ac6caca8693dd1aa5ded6039b2fed44511c71d0f670d005", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "6674261fe499272fd9c108fdb9985ae33029248ac4611f80b4119941e472c0ea", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "21730db7515c4a1c82a060385e8b90520b7c8bbcb14925a310fce679a052ac48", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "3e1463436a28fa1d12ef28451c5aa90d186d0254619e8d50080b19a4c9ea2c58", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "012cfdc64922172d68219ce2b8691f43b4f3ba44069a86e34de1b78fb8ae859f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "8f700fa730cda5f67b6184be3a54fc36a754ff2a84fee8038b3e85b3dca3eb37", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f5783241ee15a8df6aa520e97c628d9c4d17f579a1dc6f7217ed1efa65483b7b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e219-schema-normalized-32step/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "48ba5a7aaec6507a75b403d81603f3249ae5cb94b578217ee22071ed8d49e112", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e219-schema-normalized-32step/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9f39de33fb935b10028d4973f8ec0e5fd1bced8b2d07ba150d840c99a520929a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r2/checkpoints/best_ship_score.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "0ebd083535dbf7391186d8f490c54da36862a29dcc5dbc92920e9b237e3c3c7a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r2/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "fe91fd46b1b0df6acb614657706d4af256b2fbf77ef0787d945c73e42e547894", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "8b46369bc11d0828ff1c7d400db05ede3e09d927a506a2f21a7ffbc31975d936", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r3/checkpoints/best_ship_score.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "55c5064dbad0a3803f7c742e2261e03f1a155c1950e5c3f2791b4072a5ab0a09", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r3/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "4d003ae1dbf4eb2dabcb3abbd0d60a09148bbbb55bbcbe83bd08f37f0569172a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "eb2b557882bc4de7c1b9b4cdde5595a1c429c355cde52604bf6903c3489f1cc6", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/best_ship_score.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "40a157e225295263c31ec7834d78a944faa1af8d5fa0a175288852ebedf96042", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c138e55d30cb4af12721374a337babc7cb8d9cde966cd0299f0d2b0272c1e136", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9e2bebb8c625af30493e51b9c6f6aba576624d700cf8417741f8c10353fc7721", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/best_ship_score.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "89798872ec110de08bb4fcf80d23fb08e9958d5e93f47199c825444b6c3b4ed5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9caa0413522d3e5d8a3cc56f44f1ef944e229c4d5cf75aba10224c0088ef92a9", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a7851d8d4efd113153733299203922ac7164e0b2ff8b7ed33a7754aa822b168b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/best_ship_score.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "5047ca1870e801aa3345c277df5d687aeffe9694f5f2ccf6eb4d3c5f3b7137cc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "903974b672a0a83d6f1b5a88a6a6be0c0807dfd6c6273bdddfa46b5b6c2e6c4f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "2b19cbb7aec97efb61100d28b00cefb616249932a917b73cb235fd5d912bae7e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6187e8c00baad70d9491f2b9a5ffa3cbe887f04d84a86df571d094ff225cdc0d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-fastpathaux-telemetry-probe-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f7628f8166f89250140b51f2e1b01b75f7a9a19bed0bf1ca2371d5121bf60947", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-fastpathaux-telemetry-probe-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9fbf12ab764fc286864102a1d543e2bc929b5394e565f3d3e7b9f8790d653343", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-fastpathaux-telemetry-probe-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "fe09a3b68b1dc7aea76c0707ab8ed550b25bb62c42be5f4234c2dd0aabfc9f2d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-accum2-4step-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "931d8d004a0cf6d42da92fbda9978036abda2912db1e0f95730562cb755c9492", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-accum2-4step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e75b81f503ee04bae738a92afc25ce67af5396f0ed7dd70f58700fa548fd1885", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-accum2-4step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c6e4ba62b598325df995c71bbb2bd8257d6536b186500b12ba4d9858de4065f0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "27d9be59abc3420875741f34795b2da4b8122a927fa47e9e583ab28fda241a88", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f98222f96bdf0203cbee36fb5a20458b2f7924b049ec1946120f9d13a7343aa5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "cdb767271062604668cc1ac5d6981fa55da70a3c75c94ec85087cf2602ac33f8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f943321f47b5c50c687921fcfd6516aed77a97d6201903ae0a3756eba93cc0a0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1836a63c4caf2215f5a5ef8cf2b9a225454e074a37370fde80693c58bc87c745", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-seed1-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "b160ec7745b5d01fb367a5acd646a97fabd4f8cb080d4198935a175d7c9e7ea7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-seed1-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "43d21f2527f2d0270caef0ee7d6dee9861a0d367852e819281ede9f81d11aaee", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-4step-seed1-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e33b5f9596830f50cb8c724733c4ec5684d3ad96495af89000e7675df1723411", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-no-midloss-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f98222f96bdf0203cbee36fb5a20458b2f7924b049ec1946120f9d13a7343aa5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-batch8-no-midloss-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "37c98f3a5be830281f23a6a60ee7a01d4f51111389e51d02fcb7f17024d9837f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "18646410c139267f2f558147032ce5327543fd784b418e30ee9bda7e7d6c2602", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "8b48b37cdcf5528e75c77714ada34b04a48338bebd8c8bbf8d2074d07095eca0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-next-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "b2efa28eff0e54449692cc6c6ac125741b6436e892c8843f50ca060226e6e7e7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "80be2bb2b70027162f26f5a90a1816a0bf43002bfd33224f9ac147aa5e4a97ae", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1329222d34c3097e4bb2b26a5a505c7821d31d4cc4582b69dc90eeae7b4ece7f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "bd295cb96ce9dd5551d15c4205c5f96f8117b969ca518a1865fefba4e3d847fe", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f7ac95fe6f46c59f85234a8df0cc79989988c14437248bd28396055d31652e5a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c08fd319cd7b72e19990a63496dcccb5850282c11cfd9095817bd92a87848afb", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "90d365b6788679cd33908be6316f9dfca523c1e02beb15319cd86fb74b8ffe92", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1a9d63aca9ce164eb13b8f9609f8510f9f27b8aedf9786947a8e5c0ac30e3385", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6bcdb828756916d436daae07c96874d12ffcf08ff211fcec923763c433ff2ed5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3fcaff91b4bfdbb0b847958f87699f0432faf5f14ee94c49d0d29cd34cce7c6e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f788d6d0c0936d6d2c77314a880e90603889534e6bcc96b466be7bc29b44e637", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "da1cd4d759fb11c056d69f701cdf8903d2dd1f9d33c98847b25c758b4569c9d8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e0ad786c8eb50c2f7436ff8a4c2e1dd04043d06916ecfff489d93b99785982f1", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "083e0d49d5630db7fac4e0f09189449dd5fa40767db9b273c2a381d2742916b2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "33a8511d8dfaeb50072f42e29b8c5fb23d30b47012e98586491291125fafdfcd", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "52a5c61e7d739c7f670fbb3eecded0066e933d7ffc502420a2b490df958b7b17", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "738ea77137d40b023a74d450d6eaf036c0be8cbf0699f0ed516facd253c68d5b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f749c34519fdaed7b096ad1c2f977217553e5ebfb6209bc4ded262e5b4fac2e2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a22596f72aab3a1bf98a5022a8a9c132a66b46b16df7ac1609792ed9713eed60", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6c1da50b56f463e9e4782923cdf4a74e83737f8ebd303c1e775c2775bf56623d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "0cbca0cf73c33d7609881db5f832f4884faf5b6a67dcb0ac778c8edc3d98e82b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7be4ccaa98f4aa071843219ebe11ad0823c6323a1a8a49064aa43869eca9cf10", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "b57e68b1a720e35cf16f98a6f8519470e36c39b6f41d781c6bb7c9a908709bff", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f01a03aaeff363b9c4cdf8ea858653b1dd8dff0485480384f6cfb636dcef5a4f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7ef101ae1c1859c9eb6561f47e37b6dada9636a346667f1270800222b5362147", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1ea1669280c3c2fea4073e08f542867fb4aa51f85066f170218628fe69d50e7d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "306449af4a7ed734fca8f2b9f29e698759a31c29c586e139d960592c67203867", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1e27051441954b7f1f5c2fbcdf38b7d02ebac53c97452a24239b0a4b8a45187b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "03312d59833d17dfd2c07a6f3be09e6e0aa8303e5dcb1377c28921b9b4b87afc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "df2181c9a57eb341af23ff22795f32275f2c7cee6c652b6506ad2c37f63588ae", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c6c96046ab21c921edaded81a295b7126d821da3c466d69839bbb6aa62b6c627", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e8bba479cd2a002594bb4c3ffad836fcfdf7ef1c606b858c97d652988866d791", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f7982b7635b496c63ba8c09e0f6ea6127b93ede103acd268874a8848251cdfa8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "14577f9b04578758919692921c7c1048c2c34fe43b8a06fdb68e3304351a1eae", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f7982b7635b496c63ba8c09e0f6ea6127b93ede103acd268874a8848251cdfa8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "154f205d4e9232bcd366db15b46c38abd2ef21fd0a3f595b8ca6aae6c20c0e72", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "10b05f6f153a101bac1b22979e010cee506f3c313fd34f42f6ba245d7783ff7e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f814e59ba497f68284928dd9b93210e49e3ca0e8bab676fa5971e95555c01ef4", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "dacaa7df813ee99f537d4836751a8ac45d1015a0f69f427ae04f1e3e65d56841", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "bda7e7f35134f3f9a6ca1d84b7d970250f2246951c71ae7bf4b563d023fbb7bf", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c832ad3dabb4fc2416a5ec8f7fa1f49922e1b06128f12f63637d486a6709c656", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "62e00d9477b5fa44b3691fa6c1d697b518c29fc2eea19f961114d7b538027b17", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6d485bae0c32092454794f41af96abf875465f1fd82f9ff12c67c868d596609a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a22bb88120ca3dea6f51cf1d599a667817bdaea823cc199571c0b79cae65d1b4", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c46625266b218219337bcc7511acb0a8cd59dd916aa720a7510653baeccfc27d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "2755ab5f7b700dcc5cda01980df931dcae311cc41abc181a9ba49e346e99986e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "0037e12690677c3848e629816ae3d0da59fa19557e394c6110f992464500a3cc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a2c4d87e89f82b0a76133a636be08adfdc4698b4cc2482222cdb9bd97ec5b398", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "91f22e60151c7506dd7d64c3d427f4bc8466b8f8361e29ede9d884a83c79aaf7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "271d1f03bbd31a29bfc760f6c77901fb8d9efea5d54600ee841a19e9c7824df2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "2eb3efe8b9ad0a29c2b401429785035b185c7323bc6b86efc7e5341d5a1fbe50", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "df159e8ed49ad1107fca9017ce369435e991930a625ef96193289084df34f3dc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "315235ae4ef1bca1912eb5f1227004fe44061564f87e4063932f7b2fbf52027d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1ff4f214e20a122c923561d1aefc15550dfe90ee108808ab691a490f58bc4dd7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "c5ae47eed8b36ba070b76734cb503385a5a61791ebcbf2e2635bf86148c56a39", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6f1e3591829764c81cf412bd8bd0680614a517297c69c0d4fcb81aed3f0e09cf", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7aab682d7d30f9341ffd756a0cf65fba62c887ab3455f36b3118808e10d454ee", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "600d69f324fa4b6bd16d9d9ecd5d5c8f604f7fba5f26b431a22b2d6fa8918949", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-interval32-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-interval32-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "d8bb2d26e0a21108569ece5177cdef0de0feabf32bb101bad4b048619273fcc3", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "777924cad2660579359fd8b1dba1961c83ac54f0c8fb7bb8e12e483eceeb1f2c", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "2f9bcaa9c2176de6438807b9493fb3ced19a5cecd8ffa084761ce810e9a01610", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7d85b6c7265f53cca4a5353a98a908ce692ff80a7acc1c02f29e91f889452a4b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e7cc5615002773ac11c15eeeda251dd7c016e0eb03b12d9f4775b5f104c8fc86", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "e31fcfd2b03e6740ead7ce691ebf373e16de908b366b190117f25c04cf38161c", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "22c1a664f3e0540d20302b80f9bb9b7509b5ac13378c6ba10cbf9bb54cdcae58", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "25046f497d7edabe96e61af1b1b8d214387f65fe3de08a6ead297ff85987eab0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "76f4e74dbbf9ccb5d5ca0a2559384479f577c3ca5c581c41692ea9bdc480e862", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "95d8868d59df45dd7c22a27d09b7205ce375c9ef97aaa1d8070e9728cfea7237", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "d1e2ad76f953d10aba20db456b7de18ae918d6ae481aed0aee438ff9361187b1", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "bd295cb96ce9dd5551d15c4205c5f96f8117b969ca518a1865fefba4e3d847fe", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "71e95b35523236c762e25c20e226bba71f43d6916f759df6ea3fff34942e838e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7db15eff2a3a73e6372d912222667208b2bd8fc8493beb7638565e6385a792d4", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "aab63835bc695d5d0a098c2659bd793f64fd47d722480677d0ca5a551a60b805", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "23d7a9797e351cbfc8ca92bf2abbc9b2684be48648fb9836649dd2187a13c841", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "fd15d55eb8451ce089ca0299f8cbe5962a07f6fb71790dfd51c5b14c30309f4b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "b8f4abb0f5f8dd9b54b5df36cf3f4c9381c541b60250fdfe8b60af81c5c79c37", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "d210c025979c64d86b4f3dd553b3d8c0417a176ba4fb81e4ea7541106bf4d7e6", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "bd295cb96ce9dd5551d15c4205c5f96f8117b969ca518a1865fefba4e3d847fe", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "88d9e56ee589035d3adbc928cd7f7b9e62d7911b0100f0609b2153210e0da04a", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3b5281385368b5a575b0766c2c8a8407a45527535cb055751c8fd74222d624d7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3be97e80780de07a6e2ffa4f73f915fa073c40f0aecba32ab4a8f9cde33200e2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1735aaa3bf0c28c41f133648c4c83f37e372b6015263687b5574553cb4a4d398", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "80be2bb2b70027162f26f5a90a1816a0bf43002bfd33224f9ac147aa5e4a97ae", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "162f43cecf3f7e0053a7ff8d3dcf4a7fd2a520c55daba01d0bbe66ee6464f145", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "fa24f6b2f5c235514f0f01e6fff567d24054a59974e81593423d4ad2e636f075", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "db9f2227d01d01a9aefdb5274858fbe1b326a1624fc8ebba25bb9acb43a47ffc", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6b7b68b6cba2f4e4771b0103a595fb744cdfd1634e70dfa66f65f32a017f318b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f423835433c85560049881bfa8186a1c46715ecc8f6e7cad8aa24b9c65915674", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9a2e3f51500646551799945dc39cc4584658e8de01d70110845c31bada55b772", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "12c65f73f254879511a67b992c47b338571d6b86dcc4336237eb0d4abe7c35bb", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f749c34519fdaed7b096ad1c2f977217553e5ebfb6209bc4ded262e5b4fac2e2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7c835fe68c73261142df61b27541988276a103786adbe92eb5253ec689372be1", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a21e92839eea9456397c51644a22e5fcf2314356f103f584b9f5194a6f175fe8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "da923a2d8d53a60e00161d9ff5dd3cac08825f16a99349890f68858ef2042ed2", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "2274d2e172c1e90225f170ecde3c3b039f4d5965d34b13ba4ef3164caf51d778", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "21f3b7b6a54efc587eedc1e586a4331e60026c27fcb7e7e0d07a479348b27743", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9992d9539e3cd68d904889238da4f0072cdb0d2baa7abff40387e533618bc05c", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "00c6a23f363be27a1013b279411703b3872e4444f3c59baec15a67d2dd968ba8", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "3fb1817829503f2fe7b028c295729a20bddf252864d95981022f2c5f56bf14bd", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-4step-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "7874b2817373478543de0bd82dddf3688d7e0449eb1940637f49904b98e9fea6", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "00f4437b53fd11034ce59601c4a4ad2677ef2749bd9d8619595b9f1f696a09e4", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-4step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "27359580f640e7b30b8c166c6676f7a9f44c231958a8f2fd4bf8ebd22b4627e3", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "dd62dcae43bfd34874393125631b82e2fa7f40073718f05a3f360e9f01825939", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "47c0a8a01679837024abec0fa38f69ff9a7418687cfffa73383ed2486ab11797", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "1ead3aef720d766851672482c84395e8a1cad2da341c22359e32cb7920b91617", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed2-batch8-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "a980c62056893817f8dc70c2d50010192a0f3be2740d43a43a391b4782a2a014", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed3-batch8-4step-finalonly-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "55579879eb4b366e3a9b2b695ef412b4cb519c5c07a7c137eca1c0bb150ca6c9", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed3-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "ca8662d472eaa470d3ad7c9fb0004bacbb488b62d01a957272f3d35635e4bbac", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed3-batch8-4step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "393cdeb5149ca42431840041f5add286a0f3a8649e2de330f98479935d75447b", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "ed9f257bc9463368c07ea54179dbddae69ca41ab2085f7cb8aed75d7b6aadc9e", + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "f5bd15e7b5d508b398962363aff5364bef38372b60ed2c362b1dd9e4a96031d6", + "committed_sha_match": null + }, + { + "path": "outputs/runs/local_directml_adreno_20260714/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "932654f983678e376d9d4a46778b6311b69fbab498d4812fda5c046ac65b7d9d", + "committed_sha_match": null + }, + { + "path": "outputs/runs/local_directml_adreno_20260714/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "162e3916c2cf4bf269fdd1c53e0bd69c4931cefccb71dd265aaf9d198f098b36", + "committed_sha_match": null + }, + { + "path": "outputs/runs/qx_e42_factorized/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "390c2e4628064480f3607b07c12c9010016a693a20113a32d95a3c09569fbee6", + "committed_sha_match": null + }, + { + "path": "outputs/runs/qx_e42_factorized/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "dffb7f1de73553bf2385e5cd74e67576b33478d5880e097524b5698332c49e80", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm230_bounded_recursive_r4_r2/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": "1604b2cb\u20268b28a", + "actual_sha256": "1604b2cb9282928fa0969ecbbe7d78c9aa4b9907f74d0d58936bfc298a88b28a", + "committed_sha_match": true + }, + { + "path": "outputs/runs/slm230_bounded_recursive_r4_r2/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "6fb54ccc01d016e9c38445ce3aeb7cacb30adfe06efb3e437ddd48a55275fdf0", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm262_cpu_smoke_cpu_smoke/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "ec1fff0540e4cc07227013b7111d6f602b610ffc977ffaa8a4656f45606948e5", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm262_cpu_smoke_cpu_smoke/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "9cccf95a220eabf78aa1042d24adfbe78dd029e5a5490555a439ca880cbbb72f", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm262_probe/checkpoints/last.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "8727bbdc6f7b1c2d226c91f0b585f50d49e8f9bf4340f72ccc87dde8bdd29029", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm262_probe/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "90293f519bbf871ead925c6a3799dc4c18891a9abb9214a9ca73a4f43c9d46d7", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm294_tiny_baseline/checkpoints/last.pt", + "exists_locally": true, + "in_roster": true, + "committed_sha": null, + "actual_sha256": "f8017ac686f1685bd3b741b8e88f650409f9c34dd184844bfa40c198fa60beab", + "committed_sha_match": null + }, + { + "path": "outputs/runs/slm294_tiny_baseline/checkpoints/last_full_state.pt", + "exists_locally": true, + "in_roster": false, + "committed_sha": null, + "actual_sha256": "509197b823a80a6ca821022bc44d8f800d4633ec5ac31cdc126ad182782b78b1", + "committed_sha_match": null + }, + { + "path": "outputs/runs/e499-choice-compatible-strict-hf-choice-candidate-r6/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "7230ace9\u20262e2fab", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e499-remediated-roots-hf-choice-control-r4/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "bb4bec5f\u2026f359fb6", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e499-strict-r4-hf-choice-candidate-r4/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "81b2cb66\u2026bcfbaf1", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e500-document-control-hf-choice-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "a40f39a5\u2026772d6834", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e500-document-control-hf-choice-r3-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "9f752ae0\u20260b2b53", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e500-documentized-expression-hf-choice-r2/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "f54cea08\u2026773d3f0", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e500-documentized-expression-hf-choice-r4-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "a0ed6a58\u2026dda5623", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e501-e396-e500-init-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "f86b83d3\u2026cc9cf15", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e501-e396-e500-uniform-init-r2/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "14605459\u2026736e4e7", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e501-e396-e500-uniform-init-r3-1k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "d84d34c0\u2026b5be2ffd", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e502-e396-e500-prior-retained-lr3e4-r3/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "e1e833cb\u20260746cb6a", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e502-e396-e500-prior-retained-lr3e4-r4-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "6f937374\u20264a46a726", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e502-e396-e500-uniform-lr1e4-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "fcd51266\u2026f047255e", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e502-e396-e500-uniform-lr3e5-r2/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "528c86a6\u2026a62677c", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e503-e396-e500-retention0-r1-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "af6e9b1c\u20268a0af431", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e503-e396-e500-retention001-r2-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "7c5f016f\u20261be75711", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e503-e396-e500-retention003-r4-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "2dbb52db\u20265751455b", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e503-e396-e500-retention005-r3-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "4093f1aa\u2026af8d2031", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e504-e396-e500-replay000-r1-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "35cd38e0\u202656334c87", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e504-e396-e500-replay0125-r2-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "da63b403\u20261b725d3", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e504-e396-e500-replay025-r3-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "91ab3f73\u2026c3d85b4", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e504-e396-e500-replay050-r4-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "7d7e056e\u2026c90294f9", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e504-e396-e500-replay050-retention001-r5-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "1fc2fc23\u2026a36036c", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e505-e396-e500-replay050-loss-attribution-r1-5k/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "8fd11acd\u2026525967e8", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e540_training_loop_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "46c1d82d\u2026589e6014", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e541_training_loop_twotower_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "ffc8f2e3\u2026db3ffbaa", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e542-e531-root-reference-arity1-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "2d5cd4b3\u20266854c5d8", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e542_training_loop_twotower_resume_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "682ab617\u202677347c42", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e543-e531-root-reference-bounded-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "c6be3791\u202651d7f90", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e543_training_loop_twotower_resume_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "6219feed\u2026130fd1da", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e544-e543-root-identity1-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "3b6e3c00\u2026474f20c", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e544_training_loop_twotower_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "8531f7d7\u2026c2eb8036", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e545-e544-root-identity-neg1-control-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "9e54d470\u202676fa1", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e545-e544-root-identity-neg4-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "14dd4404\u202661ae", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e545_training_loop_twotower_scratch_20260719/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "3d08f592\u20261823e1fe", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e546-e544-strict-subset1-control-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "46aba904\u20260fc55", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e546-e544-strict-subset5-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "a1a6bfc9\u2026b4efe2", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e546_training_loop_twotower_scratch_20260720/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "dfc0272c\u2026351cf9c5", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e547-e544-strict-subset2-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "37002bfd\u20260fc57", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e547_training_loop_twotower_scratch_20260720/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "c67db09f\u20266d17627f", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e548_training_loop_twotower_scratch_20260720/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "6581e32f\u202681386936", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e551-e544-strict-subset2-no-lexeme-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "e7921e66\u2026dac32fc6", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e552-e544-strict-subset2-lexeme05-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "49a9c111\u2026a151fc04", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e553-e544-prior-proportional-r3-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "510e55cf\u2026e75399d", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e554-e544-slot-next-context-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "af3cbce7\u2026c67b579", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e555-e544-slot-pair-interaction-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "af53e161\u2026addf19e", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e556-e544-slot-context-combined-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "139c670c\u20265831f0a", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e557-e544-slot-pair-balance1-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "438d9871\u2026b97db05", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e558-e544-owner-coverage-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "8a572738\u2026de85382", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e558-e544-owner-coverage-r2-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "a45909df\u2026381ede", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e559-e544-owner-coverage2-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "1d11926d\u20269aac861", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e560-e544-owner-threshold4-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "dae11cee\u2026d7686a3", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e561-e544-owner-threshold7-r1-24s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "35a4fe6d\u20263a127f9", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e568-e561-cont48-r1-48s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "8dcc0804\u20260283a12b", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e569-e561-matched-cont48-r1-48s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "8254fcf7\u2026c6535f73", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e572-e569-fidelity2-r1-48s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "bb6a58ff\u2026cc29efa2", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e573-e569-fidelity1-r1-48s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "ff21fc0c\u2026cf59070d", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e574-e569-slotloss2-r1-48s/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "649cf512\u20263810b7c2", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e616-object-property-slot-bias-scratch80-20260720/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "119dd41a\u2026eef0c508", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e620-required-slot-coverage-scratch800-20260720/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "3ce5c9ef\u2026363ecc5f", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e714-symbol-only-scratch600-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "71ef1d25\u20262b49e", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e720-symbol-only-component-inventory600-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "842a1a21\u2026f91a11", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e721-symbol-only-component-plan190-r4/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "c30fd565\u202620f530", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e722-symbol-only-component-edge150-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "08873bf0\u2026a25597d", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e723-symbol-only-slot-owner140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "787d2d21\u2026bd62795", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e725-symbol-only-component-inventory130-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "897208bf\u20268ba1e", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e726-symbol-only-root-arity140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "d84148fe\u20261b91b", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e727-symbol-only-binder-arity140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "c211d2ea\u202636a857", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e729-symbol-only-binder-topology140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "c5bafb8d\u202618c43d", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e731-symbol-only-root-arity140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "bff1e0e6\u20262fbb88", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e733-symbol-only-root-identity140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "cddb5f28\u20265167fc", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/e735-symbol-only-root-arity-fullhead140-r1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "710e2dbe\u202668b970", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/gx_x22_kapur_tree_edit_s0/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": "a9cfb450e8146089cb26b6df84e90a5073627c4e59a2933d16f69034ec802ff6", + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e120-unsandboxed-20260715/e120_unsandboxed/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e121d-e53-judged-20260715/qx_e53_honest_v5_champion/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e123b-judged-20260715/e123_judged_32step_b/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e127-schema-slots-20260715/e127_judged_schema_slots/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e128-schema-slots-20260715/e128_judged_schema_slots_64/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e129-schema-slots-20260715/e129_judged_schema_slots_64_lowweights/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e130-schema-slots-20260715/e130_judged_schema_slots_seed1/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e132-generation-focus-20260715/e132_generation_focus/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e133-no-fuse-ltr-20260715/e133_no_fuse_ltr/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e135-hf-context-20260715/e135_hf_context_control/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e136-hf-context-20260715/e136_hf_context_32/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e137-hf-context-20260715/e137_hf_context_16/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e138-hf-seed1-20260715/e138_hf_context_seed1_8/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/iter-e139-hf-seed2-20260715/e139_hf_context_seed2_8/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/overnight_retrain_1000/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/overnight_retrain_200/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + }, + { + "path": "outputs/runs/restructure_cpu_scratch_v0/checkpoints/last.pt", + "exists_locally": false, + "in_roster": true, + "committed_sha": null, + "actual_sha256": null, + "committed_sha_match": null + } + ], + "scoreboards": [ + { + "scoreboard_path": "outputs/runs/e169-grammar-derived-20260716/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e169-grammar-derived-20260716/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e170-lark-state-20260716/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e170-lark-state-20260716/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e171-ast-boundary-20260716/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e171-ast-boundary-20260716/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e172-schema-types-20260716/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e172-schema-types-20260716/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-probe256/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-probe256/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-probe32/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 32, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-probe32/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 32, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-eval/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-eval/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-eval2/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-eval2/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-full/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-32step-full/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-schema-context-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-scratch-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e173-scratch-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e174-unfrozen-context-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last.pt", + "checkpoint_sha256": "15a95b415884f59ffafea3e69826a41534b15afb72aeccecb85cd2099108f3b9", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e174-unfrozen-context-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last.pt", + "checkpoint_sha256": "15a95b415884f59ffafea3e69826a41534b15afb72aeccecb85cd2099108f3b9", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e175-retrieval-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e175-retrieval-8step/checkpoints/last.pt", + "checkpoint_sha256": "ef8497ea80cd413e0c0e47f2d317ec6a331a8353ea94a11a16fd2feb682f0a34", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e175-retrieval-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e175-retrieval-8step/checkpoints/last.pt", + "checkpoint_sha256": "ef8497ea80cd413e0c0e47f2d317ec6a331a8353ea94a11a16fd2feb682f0a34", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e176-broad-corpus-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e176-broad-corpus-8step/checkpoints/last.pt", + "checkpoint_sha256": "996a5a43220b8c0216d389164b9056d8817e1fa9523ed88ff0560f6cee6c4ed1", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e176-broad-corpus-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e176-broad-corpus-8step/checkpoints/last.pt", + "checkpoint_sha256": "996a5a43220b8c0216d389164b9056d8817e1fa9523ed88ff0560f6cee6c4ed1", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e177-semantic-judge-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e177-semantic-judge-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e178-schema-arity-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e178-schema-arity-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e179-compiler-owned-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e179-compiler-owned-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e180-symbolic-root-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e180-symbolic-root-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e181-semantic-balanced-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e181-semantic-balanced-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e182-root-choice-telemetry-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e182-root-choice-telemetry-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e183-root-edge-vs-path-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e183-root-edge-vs-path-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e185-compiler-aligned-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e185-compiler-aligned-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e186-ast-complete-eos-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e186-ast-complete-eos-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e187-literal-state-sync-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e187-literal-state-sync-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e188-slot-contract-compiler-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e188-slot-contract-compiler-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e189-nesting-aware-schema-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e189-nesting-aware-schema-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e190-forward-binder-scope-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e190-forward-binder-scope-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e192-full-alignment-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "453208f977d95b1c97d1c776558ddfa8fae000151fbf99a4ef3440f443deeda0", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e192-full-alignment-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "453208f977d95b1c97d1c776558ddfa8fae000151fbf99a4ef3440f443deeda0", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e193-e184-generalized-compiler-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e193-e184-generalized-compiler-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e194-schema-symbol-scope-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e194-schema-symbol-scope-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e197-stratified-alignment-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e197-stratified-alignment-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e198-reference-complete-eos-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e198-reference-complete-eos-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e199-enum-progress-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e199-enum-progress-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e200-layout-declaration-role-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e200-layout-declaration-role-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e202-role-stratified-alignment-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e202-role-stratified-alignment-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e203-children-node-role-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e203-children-node-role-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e204-content-slot-contract-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e204-content-slot-contract-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e206-lark-terminal-alignment-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e206-lark-terminal-alignment-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e207-schema-enum-sequence-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e207-schema-enum-sequence-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e209-list-occupancy-alignment-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3bf8b8d25309818e438fd2814796edf3bd5825b454fd900ffdb80f2989615e4f", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e209-list-occupancy-alignment-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3bf8b8d25309818e438fd2814796edf3bd5825b454fd900ffdb80f2989615e4f", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e211-list-scope-occupancy-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "6674261fe499272fd9c108fdb9985ae33029248ac4611f80b4119941e472c0ea", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e211-list-scope-occupancy-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "6674261fe499272fd9c108fdb9985ae33029248ac4611f80b4119941e472c0ea", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e213-contextual-decision-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3e1463436a28fa1d12ef28451c5aa90d186d0254619e8d50080b19a4c9ea2c58", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e213-contextual-decision-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3e1463436a28fa1d12ef28451c5aa90d186d0254619e8d50080b19a4c9ea2c58", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e216-schema-role-judged-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last.pt", + "checkpoint_sha256": "8f700fa730cda5f67b6184be3a54fc36a754ff2a84fee8038b3e85b3dca3eb37", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e216-schema-role-judged-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last.pt", + "checkpoint_sha256": "8f700fa730cda5f67b6184be3a54fc36a754ff2a84fee8038b3e85b3dca3eb37", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e220-schema-normalized-probe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e219-schema-normalized-32step/checkpoints/last.pt", + "checkpoint_sha256": "48ba5a7aaec6507a75b403d81603f3249ae5cb94b578217ee22071ed8d49e112", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e220-schema-normalized-probe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e219-schema-normalized-32step/checkpoints/last.pt", + "checkpoint_sha256": "48ba5a7aaec6507a75b403d81603f3249ae5cb94b578217ee22071ed8d49e112", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e292-choice-loss-suite-frozen-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-honest-r2/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.5, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_valid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_valid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-off-r2/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval_held_out.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval_ood.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r2/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval_held_out.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval_ood.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e293-choice-component-plan-r3/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-honest-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e294-choice-no-design-control-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.25, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_valid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-honest-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e295-choice-design-dropout-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-honest-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval_held_out.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval_ood.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e296-choice-design-dropout25-r1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval_adversarial.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.25, + "row_outcomes": [ + { + "id": "adv_empty_prompt_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_deep_nest_01", + "outcome": "model_valid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "adv_many_buttons_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval_held_out.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "held_out_form_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_dual_card_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_input_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_tabs_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "held_out_settings_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval_ood.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "ood_dashboard_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_gallery_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_modal_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "ood_auth_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval_rico_held.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "rico_eval_test_25", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_42", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "rico_eval_test_77", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/e479_repro_smoke/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-chosen-only-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-chosen-only-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-fixed-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-fixed-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-postremask-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-postremask-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-remask-fixed-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-remask-fixed-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-main-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-main-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-persist-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout-persist-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout1-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-timeout1-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-top1-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-top1-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-trust-fixed-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-constrained-trust-fixed-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-explicit-lark-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-explicit-lark-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-ltr2-step64-ltr-primary-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-ltr2-step64-ltr-primary-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-attempt1/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-attempt1/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-skipprobe/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-skipprobe/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-trust/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-batch8-4step-finalonly-20260715-trust/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-profile-fixed-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-profile-fixed-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-toygrammar-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-toygrammar-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-unconstrained-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-next-unconstrained-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_abstain", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-ltr64-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-ltr64-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-one-ltr-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-one-ltr-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-topk4-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-topk4-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c08fd319cd7b72e19990a63496dcccb5850282c11cfd9095817bd92a87848afb", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c08fd319cd7b72e19990a63496dcccb5850282c11cfd9095817bd92a87848afb", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "da1cd4d759fb11c056d69f701cdf8903d2dd1f9d33c98847b25c758b4569c9d8", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "da1cd4d759fb11c056d69f701cdf8903d2dd1f9d33c98847b25c758b4569c9d8", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6bcdb828756916d436daae07c96874d12ffcf08ff211fcec923763c433ff2ed5", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6bcdb828756916d436daae07c96874d12ffcf08ff211fcec923763c433ff2ed5", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr8-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "33a8511d8dfaeb50072f42e29b8c5fb23d30b47012e98586491291125fafdfcd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-ltr8-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "33a8511d8dfaeb50072f42e29b8c5fb23d30b47012e98586491291125fafdfcd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-ltrdecode-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-ltrdecode-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7be4ccaa98f4aa071843219ebe11ad0823c6323a1a8a49064aa43869eca9cf10", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7be4ccaa98f4aa071843219ebe11ad0823c6323a1a8a49064aa43869eca9cf10", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7ef101ae1c1859c9eb6561f47e37b6dada9636a346667f1270800222b5362147", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7ef101ae1c1859c9eb6561f47e37b6dada9636a346667f1270800222b5362147", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-unique-64step-ltr2-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "1e27051441954b7f1f5c2fbcdf38b7d02ebac53c97452a24239b0a4b8a45187b", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-published-remediated-unique-64step-ltr2-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "1e27051441954b7f1f5c2fbcdf38b7d02ebac53c97452a24239b0a4b8a45187b", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-eosfix-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-eosfix-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "10b05f6f153a101bac1b22979e010cee506f3c313fd34f42f6ba245d7783ff7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "10b05f6f153a101bac1b22979e010cee506f3c313fd34f42f6ba245d7783ff7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-sample-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-diffusion-sample-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-lexer-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "bda7e7f35134f3f9a6ca1d84b7d970250f2246951c71ae7bf4b563d023fbb7bf", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-lexer-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "bda7e7f35134f3f9a6ca1d84b7d970250f2246951c71ae7bf4b563d023fbb7bf", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6d485bae0c32092454794f41af96abf875465f1fd82f9ff12c67c868d596609a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6d485bae0c32092454794f41af96abf875465f1fd82f9ff12c67c868d596609a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-256step-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "2755ab5f7b700dcc5cda01980df931dcae311cc41abc181a9ba49e346e99986e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-256step-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "2755ab5f7b700dcc5cda01980df931dcae311cc41abc181a9ba49e346e99986e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-64step-grammar-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "91f22e60151c7506dd7d64c3d427f4bc8466b8f8361e29ede9d884a83c79aaf7", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-64step-grammar-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "91f22e60151c7506dd7d64c3d427f4bc8466b8f8361e29ede9d884a83c79aaf7", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-64step-ltr2-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "df159e8ed49ad1107fca9017ce369435e991930a625ef96193289084df34f3dc", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-roots-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "df159e8ed49ad1107fca9017ce369435e991930a625ef96193289084df34f3dc", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-simple-128step-diffusion-constrained-fullsmoke-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c5ae47eed8b36ba070b76734cb503385a5a61791ebcbf2e2635bf86148c56a39", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-remediated-simple-128step-diffusion-constrained-fullsmoke-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c5ae47eed8b36ba070b76734cb503385a5a61791ebcbf2e2635bf86148c56a39", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_button_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + }, + { + "id": "smoke_callout_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-best-checkpoint-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "777924cad2660579359fd8b1dba1961c83ac54f0c8fb7bb8e12e483eceeb1f2c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-best-checkpoint-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "777924cad2660579359fd8b1dba1961c83ac54f0c8fb7bb8e12e483eceeb1f2c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-constrained-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-constrained-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-ltr2-best-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-128step-ltr2-best-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-256step-ltr2-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "e31fcfd2b03e6740ead7ce691ebf373e16de908b366b190117f25c04cf38161c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-256step-ltr2-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "e31fcfd2b03e6740ead7ce691ebf373e16de908b366b190117f25c04cf38161c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-32step-constrained-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-32step-constrained-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-32step-ltr-repair-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-32step-ltr-repair-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-constrained-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "95d8868d59df45dd7c22a27d09b7205ce375c9ef97aaa1d8070e9728cfea7237", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-constrained-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "95d8868d59df45dd7c22a27d09b7205ce375c9ef97aaa1d8070e9728cfea7237", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-diffusionmask-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7db15eff2a3a73e6372d912222667208b2bd8fc8493beb7638565e6385a792d4", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-diffusionmask-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7db15eff2a3a73e6372d912222667208b2bd8fc8493beb7638565e6385a792d4", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fastpathaux01-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fd15d55eb8451ce089ca0299f8cbe5962a07f6fb71790dfd51c5b14c30309f4b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fastpathaux01-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fd15d55eb8451ce089ca0299f8cbe5962a07f6fb71790dfd51c5b14c30309f4b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fidelity05-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fidelity05-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fidelity2-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3b5281385368b5a575b0766c2c8a8407a45527535cb055751c8fd74222d624d7", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-fidelity2-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3b5281385368b5a575b0766c2c8a8407a45527535cb055751c8fd74222d624d7", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-lexer-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-lexer-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-retrieval1-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fa24f6b2f5c235514f0f01e6fff567d24054a59974e81593423d4ad2e636f075", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-retrieval1-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fa24f6b2f5c235514f0f01e6fff567d24054a59974e81593423d4ad2e636f075", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-schema-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f423835433c85560049881bfa8186a1c46715ecc8f6e7cad8aa24b9c65915674", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-schema-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f423835433c85560049881bfa8186a1c46715ecc8f6e7cad8aa24b9c65915674", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-unfused-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr2-unfused-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr4-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "a21e92839eea9456397c51644a22e5fcf2314356f103f584b9f5194a6f175fe8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-64step-ltr4-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "a21e92839eea9456397c51644a22e5fcf2314356f103f584b9f5194a6f175fe8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-batch8-8step-unconstrained-eval-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed0-batch8-8step-unconstrained-eval-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed1-8step-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "00c6a23f363be27a1013b279411703b3872e4444f3c59baec15a67d2dd968ba8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed1-8step-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "00c6a23f363be27a1013b279411703b3872e4444f3c59baec15a67d2dd968ba8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed2-8step-feedback-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed2-8step-feedback-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed2-constrained-probe-fix-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed2-constrained-probe-fix-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed3-constrained-postfix-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "ed9f257bc9463368c07ea54179dbddae69ca41ab2085f7cb8aed75d7b6aadc9e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-seed3-constrained-postfix-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "ed9f257bc9463368c07ea54179dbddae69ca41ab2085f7cb8aed75d7b6aadc9e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-stub-eval-control-20260715/eval.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-stub-eval-control-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-timeout-control-20260715/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/iter-timeout-control-20260715/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [ + { + "id": "smoke_hero_01", + "outcome": "model_invalid", + "note": "historical row: no per-record timeout/fallback fields; timeout attribution comes from the suite-level count" + } + ] + }, + { + "scoreboard_path": "outputs/runs/slm294_tiny_baseline/eval.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/slm294_tiny_baseline/checkpoints/last.pt", + "checkpoint_sha256": "1eb2b2df1fcc8de1ede419bf70ab64d3ff4e21d355e3d6d30895f4003ce109fc", + "suite": "smoke", + "classification": "unmeasured", + "n": 0, + "decode_timeout_count": 0, + "fallback_count": null, + "decode_canvas_cap": 256, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "outputs/runs/slm294_tiny_baseline/eval_smoke.json", + "source": "local_outputs", + "checkpoint": "outputs/runs/slm294_tiny_baseline/checkpoints/last.pt", + "checkpoint_sha256": "1eb2b2df1fcc8de1ede419bf70ab64d3ff4e21d355e3d6d30895f4003ce109fc", + "suite": "smoke", + "classification": "unmeasured", + "n": 0, + "decode_timeout_count": 0, + "fallback_count": null, + "decode_canvas_cap": 256, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e120-singleton-legality-20260715.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e479-e396-array-items-full-ship-gates-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e486-json-number-typed-any-bounded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 14, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 15, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 15, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e487-json-number-typed-any-full-rico-sharded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e488-e396-json-number-typed-any-full-rico-merged-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e488-e396-json-number-typed-any-full-rico-merged-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 1500, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e489-e396-json-number-typed-any-bounded-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e490-e396-json-number-typed-any-full-ship-gates-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e497-current-main-playground-provenance-smoke-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 1, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e611-repeated-plan-slot-allocation-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e612-authored-typed-array-nonempty-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e613-schema-derived-typed-item-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e614-typed-object-state-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e615-typed-object-slot-role-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e616-object-frame-slot-bias-scratch80-replay-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": "119dd41a57573fe408ee1208a9159cafca9a45c7050bd00274571938898a8539", + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e616-object-frame-slot-bias-scratch80-replay-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": "119dd41a57573fe408ee1208a9159cafca9a45c7050bd00274571938898a8539", + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e620-required-slot-coverage-scratch800-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e620-required-slot-coverage-scratch800-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e620-required-slot-coverage-scratch800-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e621-coverage-aware-closure-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e621-coverage-aware-closure-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e621-coverage-aware-closure-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e622-coverage-closure-trace-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e622-coverage-closure-trace-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e626-required-slot-margin-decode-weight-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e626-required-slot-margin-decode-weight-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e626-required-slot-margin-decode-weight-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e628-required-slot-margin-frame-depth-exclusion-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e628-required-slot-margin-frame-depth-exclusion-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e628-required-slot-margin-frame-depth-exclusion-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e630-prompt-owned-closure-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e630-prompt-owned-closure-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e631-frame-aware-owner-escape-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e631-frame-aware-owner-escape-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e632-input-role-assignment-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e633-input-active-role-routing-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e633-input-active-role-routing-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e633-input-active-role-routing-r2-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e634-final-precontent-routing-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e635-property-compatible-coverage-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e635-property-compatible-coverage-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e636-modal-schema-reach-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e636-modal-schema-reach-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e637-nested-family-accounting-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e637-nested-family-accounting-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e638-root-slot-coverage-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e639-root-sibling-coverage-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e639-root-sibling-coverage-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e640-root-slot-references-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e641-role-plan-completion-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e642-root-only-role-plans-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e643-bound-role-plans-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e644-role-obligation-margin-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e645-root-binding-w4-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e645-root-binding-w8-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e645-root-sibling-coverage-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e645-root-sibling-coverage-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e646-complete-root-reachability-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e646-root-slot-references-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e647-role-plan-completion-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e647-root-abstention-trace-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e647-root-abstention-trace-r1-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e648-dynamic-literal-root-probe-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e648-root-only-role-plans-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e649-bound-role-plans-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e649-refresh-action-role-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e650-planned-family-role-binding-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e650-role-obligation-margin-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e651-schema-enum-literal-margin-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e652-value-text-role-20260720.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e653-nested-role-ownership-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e653-nested-role-ownership-r1-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e654-nested-role-enum-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e655-direct-role-slot-ownership-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e656-repeated-slot-role-ownership-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e657-combined-role-ownership-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e658-property-role-ownership-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e659-property-role-guard-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e666-schema-enum-finalize-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e667-nested-typed-array-owner-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e668-typed-array-role-wrapper-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e669-nested-array-schema-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e670-nested-array-role-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e671-nested-role-owner-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e672-schema-owned-array-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e673-schema-ref-reachability-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e674-semantic-role-alias-contract-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e675-schema-open-visibility-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e676-joint-role-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e678-joint-role-provenance-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e679-joint-role-smoke-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e680-display-role-aliases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e681-maximal-role-subsets-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e682-positional-role-binding-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e683-positional-role-heldout-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e684-singular-plural-family-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e685-numbered-tab-carriers-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e686-per-row-trace-budget-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e687-trace-record-identity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e688-bound-wrapper-roles-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e689-planned-child-carriers-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e690-bound-transitive-wrappers-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e691-structured-content-schema-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e692-fixed-enum-finalize-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e693-unique-group-plan-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e694-open-enum-finalize-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e695-role-property-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e696-role-literal-fallback-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e697-required-parent-order-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e698-likeness-abstention-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e699-role-capacity-revisit-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e700-namespace-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e701-schema-aware-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e701-schema-aware-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e701-schema-aware-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e701-schema-aware-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e701-schema-aware-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e702-joint-role-cardinality-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e702-joint-role-cardinality-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e702-joint-role-cardinality-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e702-joint-role-cardinality-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e702-joint-role-cardinality-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e703-enum-safe-repeated-slots-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e703-enum-safe-repeated-slots-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e703-enum-safe-repeated-slots-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e703-enum-safe-repeated-slots-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e703-enum-safe-repeated-slots-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e704-schema-value-weight-sweep-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e704-schema-value-weight-sweep-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e704-schema-value-weight-sweep-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e704-schema-value-weight-sweep-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e704-schema-value-weight-sweep-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e705-root-margin-routing-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e706-bounded-slot-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e706-bounded-slot-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e706-bounded-slot-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e706-bounded-slot-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e706-bounded-slot-carrier-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e707-carrier-root-reference-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e707-carrier-root-reference-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e707-carrier-root-reference-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e707-carrier-root-reference-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e707-carrier-root-reference-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e708-carrier-reference-obligation-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e708-carrier-reference-obligation-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e708-carrier-reference-obligation-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e708-carrier-reference-obligation-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e708-carrier-reference-obligation-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e709-final-schema-value-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e709-final-schema-value-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e709-final-schema-value-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e709-final-schema-value-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e709-final-schema-value-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e710-role-binding-negative-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e710-role-binding-negative-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e710-role-binding-negative-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e710-role-binding-negative-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e710-role-binding-negative-margin-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e711-planned-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e711-planned-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e711-planned-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e711-planned-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e711-planned-role-capacity-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e712-component-count-phrases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e712-component-count-phrases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e712-component-count-phrases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e712-component-count-phrases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e712-component-count-phrases-20260721.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e758-standalone-marker-sibling-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e759-broader-rico-prefix-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 9, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e760-e762-local-sampling-marker-uniqueness-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 8, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e763-e764-symbol-only-heldout-fallback-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "fallback_interference", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 4, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.2, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e765-e780-schema-closed-decoder-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.2, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e781-e790-numeric-literal-planning-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.6, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-e792-e802-schema-array-contract-20260722.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.6, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-sde0-01-decode-scaffolding-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-sde0-01-decode-scaffolding-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-sde0-01-decode-scaffolding-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-sde0-01-decode-scaffolding-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-sde0-01-decode-scaffolding-20260718.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-e160-lexer-32step-20260715/e160-lexer-32step/checkpoints/last.pt", + "checkpoint_sha256": "6c980d8eef3039b0766e4f50591ddb261bb32f30e5f1f08e209b52336325d699", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 32, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 32, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e173-schema-context-32step/checkpoints/last.pt", + "checkpoint_sha256": "f93bfee213cfcbff9306c6521f4ca023205bb0d58b72edc64a38092548bf0c87", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last.pt", + "checkpoint_sha256": "15a95b415884f59ffafea3e69826a41534b15afb72aeccecb85cd2099108f3b9", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e174-unfrozen-context-8step/checkpoints/last.pt", + "checkpoint_sha256": "15a95b415884f59ffafea3e69826a41534b15afb72aeccecb85cd2099108f3b9", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e175-retrieval-8step/checkpoints/last.pt", + "checkpoint_sha256": "ef8497ea80cd413e0c0e47f2d317ec6a331a8353ea94a11a16fd2feb682f0a34", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e175-retrieval-8step/checkpoints/last.pt", + "checkpoint_sha256": "ef8497ea80cd413e0c0e47f2d317ec6a331a8353ea94a11a16fd2feb682f0a34", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e176-broad-corpus-8step/checkpoints/last.pt", + "checkpoint_sha256": "996a5a43220b8c0216d389164b9056d8817e1fa9523ed88ff0560f6cee6c4ed1", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e176-broad-corpus-8step/checkpoints/last.pt", + "checkpoint_sha256": "996a5a43220b8c0216d389164b9056d8817e1fa9523ed88ff0560f6cee6c4ed1", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e177-semantic-judge-32step/checkpoints/last.pt", + "checkpoint_sha256": "fe435c622e309a2e649ce92e173834933c461d6082c5e1011630c8e07c587762", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e181-semantic-balanced-32step/checkpoints/last.pt", + "checkpoint_sha256": "40a9949b13cb81a4f0eaf6f619f54e7b8726f355c563fe43d19fab3d7bb039a3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "453208f977d95b1c97d1c776558ddfa8fae000151fbf99a4ef3440f443deeda0", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e191-full-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "453208f977d95b1c97d1c776558ddfa8fae000151fbf99a4ef3440f443deeda0", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e184-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fcf8e1434dfdaee4b115b999d163872f11f99877db93927c71ff820020b5d410", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e196-stratified-compiler-aligned-matched-32step/checkpoints/last.pt", + "checkpoint_sha256": "7afaf81defcfe192f0d567b1af086ed434ac57d0410008a9fede7c46886d897b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e201-role-stratified-compiler-aligned-32step/checkpoints/last.pt", + "checkpoint_sha256": "fd34f17eb651be9a1c54221c0ef5ca82e26d2904c88df82ecee316c3c8e5006a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e205-lark-terminal-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "34a04c836d81783728289715fdc1d92ae6632b5d05827860dac2af6563616efa", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3bf8b8d25309818e438fd2814796edf3bd5825b454fd900ffdb80f2989615e4f", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e208-list-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3bf8b8d25309818e438fd2814796edf3bd5825b454fd900ffdb80f2989615e4f", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "6674261fe499272fd9c108fdb9985ae33029248ac4611f80b4119941e472c0ea", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e210-list-scope-occupancy-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "6674261fe499272fd9c108fdb9985ae33029248ac4611f80b4119941e472c0ea", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3e1463436a28fa1d12ef28451c5aa90d186d0254619e8d50080b19a4c9ea2c58", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e212-contextual-decision-stratified-32step/checkpoints/last.pt", + "checkpoint_sha256": "3e1463436a28fa1d12ef28451c5aa90d186d0254619e8d50080b19a4c9ea2c58", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last.pt", + "checkpoint_sha256": "8f700fa730cda5f67b6184be3a54fc36a754ff2a84fee8038b3e85b3dca3eb37", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e215-schema-role-judged-32step/checkpoints/last.pt", + "checkpoint_sha256": "8f700fa730cda5f67b6184be3a54fc36a754ff2a84fee8038b3e85b3dca3eb37", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e219-schema-normalized-32step/checkpoints/last.pt", + "checkpoint_sha256": "48ba5a7aaec6507a75b403d81603f3249ae5cb94b578217ee22071ed8d49e112", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e219-schema-normalized-32step/checkpoints/last.pt", + "checkpoint_sha256": "48ba5a7aaec6507a75b403d81603f3249ae5cb94b578217ee22071ed8d49e112", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/ladders/e292-choice-loss-suite-complete-r2/runs/capacity_choice_v1__d64_h2_c1_dn2_t5000_x1__s0/checkpoints/last.pt", + "checkpoint_sha256": "7cad143139f91369b4780878f634e7fa24434d7ccb8f9813a00f5412f3051c99", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r2/checkpoints/last.pt", + "checkpoint_sha256": "8c6b7595373d623cc821ddfb7a362faaedd4f3aa87bf360133fb167889a9157d", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e293-choice-component-plan-r3/checkpoints/last.pt", + "checkpoint_sha256": "78b70c81bd16395e22718baa91b50427c205f38136269c6248b85562cdec5308", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e294-choice-no-design-control-r1/checkpoints/last.pt", + "checkpoint_sha256": "df30ca03f8f2bc3313b1b8afff9c40b7ab18c4fd2b0e8ae1b3888ba780d9add0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e295-choice-design-dropout-r1/checkpoints/last.pt", + "checkpoint_sha256": "5b4c50467454f7a9dddbc28da2e115c31a8eba8071587e95eda096729a16fb50", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e296-choice-design-dropout25-r1/checkpoints/last.pt", + "checkpoint_sha256": "b3c4df4cca25905d1101ed8006f430a772a7228f894530ef98cb8fd8cfc1a1ed", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "adversarial", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "held_out", + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "ood", + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "rico_held", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/e396-balanced-type-head-continuation-r1/checkpoints/last.pt", + "checkpoint_sha256": "feefa0564490bd1db42f79ff710143ad8ed07ab9e4e324f2744a30f8c2f2eee0", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-next-batch8-4step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "3f5a50e751b9ac81162baf3e7cdeabb97349ba11b4826a6cc818fcbf23d61eee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 128, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 3, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c08fd319cd7b72e19990a63496dcccb5850282c11cfd9095817bd92a87848afb", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-firsttoken-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c08fd319cd7b72e19990a63496dcccb5850282c11cfd9095817bd92a87848afb", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "da1cd4d759fb11c056d69f701cdf8903d2dd1f9d33c98847b25c758b4569c9d8", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-fixed-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "da1cd4d759fb11c056d69f701cdf8903d2dd1f9d33c98847b25c758b4569c9d8", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6bcdb828756916d436daae07c96874d12ffcf08ff211fcec923763c433ff2ed5", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr2-mixedmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6bcdb828756916d436daae07c96874d12ffcf08ff211fcec923763c433ff2ed5", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "33a8511d8dfaeb50072f42e29b8c5fb23d30b47012e98586491291125fafdfcd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-ltr8-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "33a8511d8dfaeb50072f42e29b8c5fb23d30b47012e98586491291125fafdfcd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-64step-unfused-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f1f40d72e9cba23289e873fba98b53c473e904ea03e7819b1af44829994c0bf9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7be4ccaa98f4aa071843219ebe11ad0823c6323a1a8a49064aa43869eca9cf10", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-cap3-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7be4ccaa98f4aa071843219ebe11ad0823c6323a1a8a49064aa43869eca9cf10", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7ef101ae1c1859c9eb6561f47e37b6dada9636a346667f1270800222b5362147", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-organic-balanced-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7ef101ae1c1859c9eb6561f47e37b6dada9636a346667f1270800222b5362147", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "1e27051441954b7f1f5c2fbcdf38b7d02ebac53c97452a24239b0a4b8a45187b", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-published-remediated-unique-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "1e27051441954b7f1f5c2fbcdf38b7d02ebac53c97452a24239b0a4b8a45187b", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fastpathaux1-telemetry-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "487ca5bd3d16cb25c107270b29fc283efc59489030f584b9264445f12d84c59f", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "10b05f6f153a101bac1b22979e010cee506f3c313fd34f42f6ba245d7783ff7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "10b05f6f153a101bac1b22979e010cee506f3c313fd34f42f6ba245d7783ff7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "511553d33630d96a11b9921f0a0176d73030f794a737761c712bcf4ec800a0f9", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "bda7e7f35134f3f9a6ca1d84b7d970250f2246951c71ae7bf4b563d023fbb7bf", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "bda7e7f35134f3f9a6ca1d84b7d970250f2246951c71ae7bf4b563d023fbb7bf", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6d485bae0c32092454794f41af96abf875465f1fd82f9ff12c67c868d596609a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-128step-retrieval1-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "6d485bae0c32092454794f41af96abf875465f1fd82f9ff12c67c868d596609a", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "2755ab5f7b700dcc5cda01980df931dcae311cc41abc181a9ba49e346e99986e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-256step-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "2755ab5f7b700dcc5cda01980df931dcae311cc41abc181a9ba49e346e99986e", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "91f22e60151c7506dd7d64c3d427f4bc8466b8f8361e29ede9d884a83c79aaf7", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-grammar-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "91f22e60151c7506dd7d64c3d427f4bc8466b8f8361e29ede9d884a83c79aaf7", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "df159e8ed49ad1107fca9017ce369435e991930a625ef96193289084df34f3dc", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-roots-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "df159e8ed49ad1107fca9017ce369435e991930a625ef96193289084df34f3dc", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c5ae47eed8b36ba070b76734cb503385a5a61791ebcbf2e2635bf86148c56a39", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-remediated-simple-128step-diffusion-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "c5ae47eed8b36ba070b76734cb503385a5a61791ebcbf2e2635bf86148c56a39", + "suite": "smoke", + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 64, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "777924cad2660579359fd8b1dba1961c83ac54f0c8fb7bb8e12e483eceeb1f2c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-interval32-testdir-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "777924cad2660579359fd8b1dba1961c83ac54f0c8fb7bb8e12e483eceeb1f2c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "07a16a3e232b0d2c8ddb6370562c86d38469cf5c6e09e497e526900c8c33ce7e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-128step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "e31fcfd2b03e6740ead7ce691ebf373e16de908b366b190117f25c04cf38161c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-256step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "e31fcfd2b03e6740ead7ce691ebf373e16de908b366b190117f25c04cf38161c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-32step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "006ea99a721ed37702a28d35ed03dc1323e487fafc44dd2890f8bc4199545fad", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "95d8868d59df45dd7c22a27d09b7205ce375c9ef97aaa1d8070e9728cfea7237", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "95d8868d59df45dd7c22a27d09b7205ce375c9ef97aaa1d8070e9728cfea7237", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7db15eff2a3a73e6372d912222667208b2bd8fc8493beb7638565e6385a792d4", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-diffusionmask-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "7db15eff2a3a73e6372d912222667208b2bd8fc8493beb7638565e6385a792d4", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fd15d55eb8451ce089ca0299f8cbe5962a07f6fb71790dfd51c5b14c30309f4b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fastpathaux01-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fd15d55eb8451ce089ca0299f8cbe5962a07f6fb71790dfd51c5b14c30309f4b", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity05-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3b5281385368b5a575b0766c2c8a8407a45527535cb055751c8fd74222d624d7", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-fidelity2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3b5281385368b5a575b0766c2c8a8407a45527535cb055751c8fd74222d624d7", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-lexer-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fa24f6b2f5c235514f0f01e6fff567d24054a59974e81593423d4ad2e636f075", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-retrieval1-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "fa24f6b2f5c235514f0f01e6fff567d24054a59974e81593423d4ad2e636f075", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f423835433c85560049881bfa8186a1c46715ecc8f6e7cad8aa24b9c65915674", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-schema-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "f423835433c85560049881bfa8186a1c46715ecc8f6e7cad8aa24b9c65915674", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr2-unfused-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "3ae0d8155a29d04a9755b15bcd399d872fe75cf84258d61fab871415ce1c95cd", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "a21e92839eea9456397c51644a22e5fcf2314356f103f584b9f5194a6f175fe8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-64step-ltr4-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "a21e92839eea9456397c51644a22e5fcf2314356f103f584b9f5194a6f175fe8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "00c6a23f363be27a1013b279411703b3872e4444f3c59baec15a67d2dd968ba8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed1-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "00c6a23f363be27a1013b279411703b3872e4444f3c59baec15a67d2dd968ba8", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed2-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "8447d430ee46023479de9a076c6ddb45bf06f20f65d9a46c1a71ed03f68630ee", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "ed9f257bc9463368c07ea54179dbddae69ca41ab2085f7cb8aed75d7b6aadc9e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed3-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "ed9f257bc9463368c07ea54179dbddae69ca41ab2085f7cb8aed75d7b6aadc9e", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": null, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/iter-seed0-batch8-8step-finalonly-20260715/checkpoints/last.pt", + "checkpoint_sha256": "f4927e68bb15183e0b168499e8b0bd1bcebdbd19b75fb8d93e0410cd9f9a1fb3", + "suite": "smoke", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/slm294_tiny_baseline/checkpoints/last.pt", + "checkpoint_sha256": "1eb2b2df1fcc8de1ede419bf70ab64d3ff4e21d355e3d6d30895f4003ce109fc", + "suite": "smoke", + "classification": "unmeasured", + "n": 0, + "decode_timeout_count": 0, + "fallback_count": null, + "decode_canvas_cap": 256, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": "outputs/runs/slm294_tiny_baseline/checkpoints/last.pt", + "checkpoint_sha256": "1eb2b2df1fcc8de1ede419bf70ab64d3ff4e21d355e3d6d30895f4003ce109fc", + "suite": "smoke", + "classification": "unmeasured", + "n": 0, + "decode_timeout_count": 0, + "fallback_count": null, + "decode_canvas_cap": 256, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": "rico_held", + "classification": "model_behavior", + "n": 1, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 48, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 24, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 16, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 14, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 15, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 15, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 1500, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "runtime_timeout_interference", + "n": 3, + "decode_timeout_count": 1, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 0.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": "119dd41a57573fe408ee1208a9159cafca9a45c7050bd00274571938898a8539", + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": "119dd41a57573fe408ee1208a9159cafca9a45c7050bd00274571938898a8539", + "suite": null, + "classification": "model_behavior", + "n": null, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.25, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.8, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.5, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 0.75, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": null, + "meaningful_program_rate": null, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 9, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 8, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 1.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "fallback_interference", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 4, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.2, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.2, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.6, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.6, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 5, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 4, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + }, + { + "scoreboard_path": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "source": "docs_design_historical", + "checkpoint": null, + "checkpoint_sha256": null, + "suite": null, + "classification": "model_behavior", + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": null, + "parse_rate": 1.0, + "meaningful_program_rate": 0.0, + "row_outcomes": [] + } + ], + "hash_pins": { + "ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "actual_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "scoreboard_recorded_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "matches_scoreboard_record": true + }, + "lexer_ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "exists_locally": true, + "actual_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "scoreboard_recorded_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "matches_scoreboard_record": true + } + }, + "summary": { + "n_local_checkpoints": 216, + "n_roster_rows": 115, + "n_committed_sha_verified": 1, + "n_committed_sha_mismatch": 0, + "n_unverifiable": 305, + "scoreboard_classes": { + "fallback_interference": 2, + "model_behavior": 1134, + "runtime_timeout_interference": 10, + "unmeasured": 4 + } + } + }, + "sweep": { + "n_cells": 12, + "pairs": [ + { + "checkpoint": "lexer_ltr2", + "record_id": "smoke_button_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 2091.33, + "elapsed_ms_10x": 1683.37 + }, + { + "checkpoint": "lexer_ltr2", + "record_id": "smoke_callout_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 1643.82, + "elapsed_ms_10x": 1693.5 + }, + { + "checkpoint": "lexer_ltr2", + "record_id": "smoke_hero_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 1901.12, + "elapsed_ms_10x": 1589.12 + }, + { + "checkpoint": "ltr2", + "record_id": "smoke_button_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 1276.21, + "elapsed_ms_10x": 1519.6 + }, + { + "checkpoint": "ltr2", + "record_id": "smoke_callout_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 1196.94, + "elapsed_ms_10x": 1842.18 + }, + { + "checkpoint": "ltr2", + "record_id": "smoke_hero_01", + "status": "paired", + "pair_violation_fields": [], + "budget_baseline": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "budget_10x": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "outcome_baseline": null, + "outcome_10x": null, + "cell_status_baseline": "not_rerunnable", + "cell_status_10x": "not_rerunnable", + "parse_ok_baseline": null, + "parse_ok_10x": null, + "elapsed_ms_baseline": 1217.16, + "elapsed_ms_10x": 2402.0 + } + ], + "outcome_counts": { + "baseline": { + "model_valid": 0, + "model_invalid": 0, + "model_abstain": 0, + "runtime_timeout": 0, + "fallback_output": 0, + "harness_error": 0 + }, + "budget10x": { + "model_valid": 0, + "model_invalid": 0, + "model_abstain": 0, + "runtime_timeout": 0, + "fallback_output": 0, + "harness_error": 0 + } + }, + "cells": [ + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 2091.33, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_button_01", + "record_sha256": "2caa4109b55685eda55860df59afb5d16b51ce55399ae1411416709f8ffe1e82", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1643.82, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_callout_01", + "record_sha256": "ddab24fb33b82c665cd22bbfffd7895b4d425a7fade5fd0bb6fadb79effe91b3", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1901.12, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 128, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_hero_01", + "record_sha256": "b44a979b99563d2c5744ba63612f05c6231cbb064593bf5da021f48fcb27605e", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1683.37, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_button_01", + "record_sha256": "2caa4109b55685eda55860df59afb5d16b51ce55399ae1411416709f8ffe1e82", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1693.5, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_callout_01", + "record_sha256": "ddab24fb33b82c665cd22bbfffd7895b4d425a7fade5fd0bb6fadb79effe91b3", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1589.12, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "lexer_ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 1280, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_hero_01", + "record_sha256": "b44a979b99563d2c5744ba63612f05c6231cbb064593bf5da021f48fcb27605e", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1276.21, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_button_01", + "record_sha256": "2caa4109b55685eda55860df59afb5d16b51ce55399ae1411416709f8ffe1e82", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1196.94, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_callout_01", + "record_sha256": "ddab24fb33b82c665cd22bbfffd7895b4d425a7fade5fd0bb6fadb79effe91b3", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1217.16, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "baseline", + "budget": { + "timeout_s": 10.0, + "canvas_cap": 256, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_hero_01", + "record_sha256": "b44a979b99563d2c5744ba63612f05c6231cbb064593bf5da021f48fcb27605e", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1519.6, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_button_01", + "record_sha256": "2caa4109b55685eda55860df59afb5d16b51ce55399ae1411416709f8ffe1e82", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 1842.18, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_callout_01", + "record_sha256": "ddab24fb33b82c665cd22bbfffd7895b4d425a7fade5fd0bb6fadb79effe91b3", + "seed": 0, + "outcome_record": null + }, + { + "status": "not_rerunnable", + "outcome": null, + "stop_reason": "not_rerunnable", + "elapsed_ms": 2402.0, + "prediction": null, + "parse_ok": null, + "error": "checkpoint output contract v0 is incompatible with required symbol_only/v2; retrain from symbol-only targets", + "schema": "slm303_sweep_cell/v1", + "checkpoint": "ltr2", + "checkpoint_path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "checkpoint_sha256": "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a", + "arm": "budget10x", + "budget": { + "timeout_s": 100.0, + "canvas_cap": 2560, + "gen_steps": null, + "max_attempts": 1 + }, + "record_id": "smoke_hero_01", + "record_sha256": "b44a979b99563d2c5744ba63612f05c6231cbb064593bf5da021f48fcb27605e", + "seed": 0, + "outcome_record": null + } + ] + }, + "disposition": { + "annotations": [ + { + "scoreboard": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-e497-current-main-playground-provenance-smoke-20260718.json", + "annotation": "runtime/harness artifact: 1/3 decode timeouts at canvas cap None; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "runtime/harness artifact: 1/3 decode timeouts at canvas cap None; zero parse rate on these rows is budget interference, NOT a model verdict" + }, + { + "scoreboard": "docs/design/iter-e763-e764-symbol-only-heldout-fallback-20260722.json", + "annotation": "fallback interference: fallback_count=4; fallback outputs never classify as model success" + }, + { + "scoreboard": "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "annotation": "fallback interference: fallback_count=4; fallback outputs never classify as model success" + } + ], + "counts": { + "runtime_timeout_interference": 10, + "fallback_interference": 2, + "unmeasured": 4 + }, + "sweep_verdict": "not_rerunnable: 6/6 paired cells blocked by the output-contract gate (checkpoint v0 vs required symbol_only/v2, no migration path); the 10x-budget flip question is UNANSWERED by re-decode \u2014 census + taxonomy carry the audit", + "recommendation": { + "powered_rerun": "at most one: retrain the remediated recipe from symbol-only targets (output contract v2 \u2014 the v0 checkpoints are blocked by require_current_output_contract with no migration path), then a preregistered full-smoke re-eval at recorded vs 10x decode budget with per-record decode_outcome fields (now emitted by eval_runner), n\u226516 smoke+fixture records for Wilson resolution", + "rationale": "census localizes all timeout interference to the two remediated checkpoints; no other hash-verifiable checkpoint carries a timeout-flagged scoreboard, and re-decode of the v0 checkpoints under current code is impossible" + } + }, + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "cc2521119d6d89d4663131a7001ce45897ca8cdd", + "code_dirty": true, + "components": { + "harness.experiments.slm303_decode_budget_audit": "v1", + "harness.model_build.eval": "v51" + }, + "stamped_at": "2026-07-24T19:08:59.985574+00:00" + } +} diff --git a/docs/design/iter-slm303-decode-budget-audit-20260724.md b/docs/design/iter-slm303-decode-budget-audit-20260724.md new file mode 100644 index 000000000..1726171cf --- /dev/null +++ b/docs/design/iter-slm303-decode-budget-audit-20260724.md @@ -0,0 +1,54 @@ +# SLM-303 (LAR1-05): decode-budget audit — census, 10x sweep, disposition + +- generated_at: `2026-07-24T19:08:59Z` +- claim_class: `diagnostic` (diagnostic; not ship evidence) +- taxonomy: `model_valid, model_invalid, model_abstain, runtime_timeout, fallback_output, harness_error` +- budget evidence: inferred from recorded eval_smoke.json: all 3 smoke rows share latency_ms=3333.79 (one chunk of 3 killed at ~10001ms → 10s timeout); the original CLI flag was not persisted (config=null), disclosed as inferred + +## Census + +- local checkpoints enumerated: 216 (roster rows: 115) +- committed SHA verified: 1; mismatch: 0; unverifiable (no committed SHA): 305 +- scoreboard classes: `{"fallback_interference": 2, "model_behavior": 1134, "runtime_timeout_interference": 10, "unmeasured": 4}` + +### Hash pins (sweep checkpoints) + +| key | path | sha256 | matches scoreboard record | +| --- | --- | --- | --- | +| ltr2 | `outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt` | `653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a` | True | +| lexer_ltr2 | `outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt` | `4573bf7d0f606e9f91471a402988ebdcbb7596cf8425ac95fea54866f360310c` | True | + +## Budget sweep (preregistered, paired) + +| checkpoint | record | baseline outcome | 10x outcome | baseline ms | 10x ms | +| --- | --- | --- | --- | --- | --- | +| lexer_ltr2 | smoke_button_01 | not_rerunnable | not_rerunnable | 2091.33 | 1683.37 | +| lexer_ltr2 | smoke_callout_01 | not_rerunnable | not_rerunnable | 1643.82 | 1693.5 | +| lexer_ltr2 | smoke_hero_01 | not_rerunnable | not_rerunnable | 1901.12 | 1589.12 | +| ltr2 | smoke_button_01 | not_rerunnable | not_rerunnable | 1276.21 | 1519.6 | +| ltr2 | smoke_callout_01 | not_rerunnable | not_rerunnable | 1196.94 | 1842.18 | +| ltr2 | smoke_hero_01 | not_rerunnable | not_rerunnable | 1217.16 | 2402.0 | + +- outcome counts (baseline): `{"model_valid": 0, "model_invalid": 0, "model_abstain": 0, "runtime_timeout": 0, "fallback_output": 0, "harness_error": 0}` +- outcome counts (10x): `{"model_valid": 0, "model_invalid": 0, "model_abstain": 0, "runtime_timeout": 0, "fallback_output": 0, "harness_error": 0}` + +## Disposition (append-only annotations) + +- `outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict +- `outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval_smoke.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict +- `outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict +- `outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-e497-current-main-playground-provenance-smoke-20260718.json`: runtime/harness artifact: 1/3 decode timeouts at canvas cap None; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 128; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: runtime/harness artifact: 3/3 decode timeouts at canvas cap 256; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: runtime/harness artifact: 1/3 decode timeouts at canvas cap None; zero parse rate on these rows is budget interference, NOT a model verdict +- `docs/design/iter-e763-e764-symbol-only-heldout-fallback-20260722.json`: fallback interference: fallback_count=4; fallback outputs never classify as model success +- `docs/design/iter-slm303-decode-budget-audit-20260724.json`: fallback interference: fallback_count=4; fallback outputs never classify as model success + +- sweep verdict: not_rerunnable: 6/6 paired cells blocked by the output-contract gate (checkpoint v0 vs required symbol_only/v2, no migration path); the 10x-budget flip question is UNANSWERED by re-decode — census + taxonomy carry the audit +- powered-rerun recommendation: at most one: retrain the remediated recipe from symbol-only targets (output contract v2 — the v0 checkpoints are blocked by require_current_output_contract with no migration path), then a preregistered full-smoke re-eval at recorded vs 10x decode budget with per-record decode_outcome fields (now emitted by eval_runner), n≥16 smoke+fixture records for Wilson resolution + - rationale: census localizes all timeout interference to the two remediated checkpoints; no other hash-verifiable checkpoint carries a timeout-flagged scoreboard, and re-decode of the v0 checkpoints under current code is impossible + +Historical iter docs and MODEL_CARD rows are untouched; these annotations are additive. diff --git a/scripts/run_slm303_decode_budget_audit.py b/scripts/run_slm303_decode_budget_audit.py new file mode 100644 index 000000000..eaf8e9b72 --- /dev/null +++ b/scripts/run_slm303_decode_budget_audit.py @@ -0,0 +1,917 @@ +#!/usr/bin/env python3 +"""SLM-303 (LAR1-05): decode-budget audit — census, budget sweep, disposition. + +Question: which historical zero / interference-flagged scoreboards are model +behavior, and which are runtime/harness budget artifacts (decode timeouts, +fallbacks)? Answered three ways: + +1. ``census`` — enumerate every local ``outputs/runs/*/checkpoints/*.pt`` plus + every MODEL_CARD roster row; verify committed SHA fragments + (prefix…suffix) against actual sha256 of local bytes; group every local + ``eval_*.json`` scoreboard (and committed docs/design iter JSONs for + historical context) by checkpoint; classify each scoreboard with the + SLM-303 decode-outcome taxonomy (``unmeasured`` when fallback_count is + null). Hash-pins the two remediated checkpoints that carry the only + nonzero-timeout scoreboards. +2. ``sweep-cell`` / ``report`` — PREREGISTERED budget sweep: same checkpoint, + same 3 smoke records (test_seeds smoke ids — the original suite directory + is empty, so the suite is rebuilt from test_seeds; disclosed), same seed, + checkpoint-declared decode policy; arms differ ONLY in budget: baseline + (recorded canvas cap 256/128, inferred 10s decode timeout from the recorded + 3×3333.79ms chunk latency) vs 10x (100s, 2560/1280). Each + (checkpoint, arm, record) cell runs in its own hard-killed worker process + so every command stays under the run cap; a killed cell is ``decode_timeout`` + evidence, never a fabricated result. +3. ``report`` — disposition: append-only historical annotations + a small + powered-rerun recommendation, written to + ``docs/design/iter-slm303-decode-budget-audit-20260724.{json,md}``. + +Example: + python -m scripts.run_slm303_decode_budget_audit census + python -m scripts.run_slm303_decode_budget_audit sweep-cell \ + --checkpoint ltr2 --arm baseline --record-id smoke_hero_01 + python -m scripts.run_slm303_decode_budget_audit report +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from slm_training.harnesses.model_build.decode_outcome import ( + DECODE_OUTCOMES, + DecodeOutcomeRecordV1, + classify_decode_outcome, + outcome_counts, +) +from slm_training.versioning import build_version_stamp + +COMPONENT = "harness.experiments.slm303_decode_budget_audit" +EVAL_COMPONENT = "harness.model_build.eval" + +RUNS_ROOT = Path("outputs/runs") +MODEL_CARD = Path("docs/MODEL_CARD.md") +DESIGN_DIR = Path("docs/design") +TEST_SEEDS = Path("src/slm_training/resources/test_seeds.jsonl") + +AUDIT_DIR = RUNS_ROOT / "slm303_decode_budget_audit" +CENSUS_JSON = AUDIT_DIR / "census.json" +CELLS_DIR = AUDIT_DIR / "cells" + +DEFAULT_JSON_OUT = DESIGN_DIR / "iter-slm303-decode-budget-audit-20260724.json" +DEFAULT_MD_OUT = DESIGN_DIR / "iter-slm303-decode-budget-audit-20260724.md" + +# The only two checkpoints with ACTUAL nonzero-timeout scoreboards. Canvas +# caps and timeout evidence come from their recorded eval_smoke.json. +SWEEP_CHECKPOINTS: dict[str, dict[str, Any]] = { + "ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "scoreboard": "outputs/runs/iter-published-remediated-64step-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "recorded_canvas_cap": 256, + "recorded_decode_timeout_count": 3, + }, + "lexer_ltr2": { + "path": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-20260715/checkpoints/best_weighted_nll.pt", + "scoreboard": "outputs/runs/iter-published-remediated-64step-lexer-ltr2-constrained-fullsmoke-20260715/eval_smoke.json", + "recorded_canvas_cap": 128, + "recorded_decode_timeout_count": 3, + }, +} + +# The recorded timeout scoreboards show latency_ms = 3333.79 on all 3 smoke +# records: one chunk of 3 killed at ~10001ms → decode_timeout_seconds = 10. +BASELINE_TIMEOUT_S = 10.0 +BASELINE_TIMEOUT_EVIDENCE = ( + "inferred from recorded eval_smoke.json: all 3 smoke rows share " + "latency_ms=3333.79 (one chunk of 3 killed at ~10001ms → 10s timeout); " + "the original CLI flag was not persisted (config=null), disclosed as inferred" +) +SWEEP_BUDGET_MULTIPLIER = 10 + +SMOKE_RECORD_IDS = ("smoke_hero_01", "smoke_button_01", "smoke_callout_01") + +_SHA_FRAGMENT_RE = re.compile(r"\b([0-9a-f]{6,})…([0-9a-f]{4,})\b") +_SHA_FULL_RE = re.compile(r"\b[0-9a-f]{64}\b") +_CHECKPOINT_PATH_RE = re.compile(r"outputs/runs/\S*?checkpoints/[\w.\-]+\.pt") + + +# --------------------------------------------------------------------------- +# Pure helpers (unit-tested) +# --------------------------------------------------------------------------- + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + for block in iter(lambda: fh.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def sha_matches_committed(committed: str | None, actual: str) -> bool | None: + """Verify a committed SHA reference against actual bytes. + + ``committed`` may be a full 64-hex digest or a ``prefix…suffix`` fragment + (as used in MODEL_CARD roster rows). Returns None when no committed + reference exists (unverifiable, never a pass). + """ + if not committed: + return None + committed = committed.strip().strip("`") + if _SHA_FULL_RE.fullmatch(committed): + return committed == actual + match = _SHA_FRAGMENT_RE.search(committed) + if not match: + return None + prefix, suffix = match.groups() + return actual.startswith(prefix) and actual.endswith(suffix) + + +def classify_scoreboard(scoreboard: dict[str, Any]) -> dict[str, Any]: + """Classify one scoreboard dict with the decode-outcome taxonomy. + + Scoreboard-level view: ``unmeasured`` when fallback telemetry is absent + (fallback_count null), ``runtime_timeout_interference`` when decode + timeouts fired, ``fallback_interference`` when fallbacks fired, else + ``model_behavior`` — a clean zero is a model verdict, not a budget + artifact. Per-record rows are classified when details[] exists. + """ + fallback_count = scoreboard.get("fallback_count") + timeout_count = scoreboard.get("decode_timeout_count") + n = scoreboard.get("n") + row_outcomes: list[dict[str, Any]] = [] + for row in scoreboard.get("details") or []: + if not isinstance(row, dict) or "id" not in row: + continue + row_outcomes.append( + { + "id": row.get("id"), + "outcome": classify_decode_outcome( + parse_ok=row.get("parse_ok"), + error=row.get("error"), + fallback_counters=int(row.get("fallback_count") or 0), + timed_out=False, # historical rows carry no timeout fact + abstained=not str(row.get("prediction") or "").strip(), + harness_exception=False, + ), + "note": ( + "historical row: no per-record timeout/fallback fields; " + "timeout attribution comes from the suite-level count" + ), + } + ) + if fallback_count is None: + classification = "unmeasured" + elif timeout_count: + classification = "runtime_timeout_interference" + elif fallback_count: + classification = "fallback_interference" + else: + classification = "model_behavior" + return { + "classification": classification, + "n": n, + "decode_timeout_count": timeout_count, + "fallback_count": fallback_count, + "decode_canvas_cap": scoreboard.get("decode_canvas_cap"), + "parse_rate": scoreboard.get("parse_rate"), + "meaningful_program_rate": scoreboard.get("meaningful_program_rate"), + "row_outcomes": row_outcomes, + } + + +def build_sweep_arms(recorded_canvas_cap: int) -> dict[str, dict[str, Any]]: + """Preregistered arms; they differ ONLY in budget fields.""" + return { + "baseline": { + "timeout_s": BASELINE_TIMEOUT_S, + "canvas_cap": recorded_canvas_cap, + "timeout_evidence": BASELINE_TIMEOUT_EVIDENCE, + }, + "budget10x": { + "timeout_s": BASELINE_TIMEOUT_S * SWEEP_BUDGET_MULTIPLIER, + "canvas_cap": recorded_canvas_cap * SWEEP_BUDGET_MULTIPLIER, + "timeout_evidence": "10x the preregistered baseline budget", + }, + } + + +def pair_cells( + cells: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Pair baseline vs budget10x cells per (checkpoint, record); assert the + arms differ only in budget fields (paired-design check).""" + pairs: list[dict[str, Any]] = [] + by_key: dict[tuple[str, str], dict[str, Any]] = {} + for cell in cells: + by_key.setdefault((cell["checkpoint"], cell["record_id"]), {})[ + cell["arm"] + ] = cell + for (checkpoint, record_id), arms in sorted(by_key.items()): + base = arms.get("baseline") + wide = arms.get("budget10x") + if base is None or wide is None: + pairs.append( + { + "checkpoint": checkpoint, + "record_id": record_id, + "status": "unpaired", + "arms_present": sorted(arms), + } + ) + continue + base_budget = base["budget"] + wide_budget = wide["budget"] + non_budget_mismatch = [ + field + for field in ("checkpoint_sha256", "seed", "record_sha256") + if base.get(field) != wide.get(field) + ] + pairs.append( + { + "checkpoint": checkpoint, + "record_id": record_id, + "status": "paired" if not non_budget_mismatch else "pair_violation", + "pair_violation_fields": non_budget_mismatch, + "budget_baseline": base_budget, + "budget_10x": wide_budget, + "outcome_baseline": base["outcome"], + "outcome_10x": wide["outcome"], + "cell_status_baseline": base.get("status"), + "cell_status_10x": wide.get("status"), + "parse_ok_baseline": base.get("parse_ok"), + "parse_ok_10x": wide.get("parse_ok"), + "elapsed_ms_baseline": base.get("elapsed_ms"), + "elapsed_ms_10x": wide.get("elapsed_ms"), + } + ) + return pairs + + +# --------------------------------------------------------------------------- +# Census +# --------------------------------------------------------------------------- + + +def _roster_rows(text: str) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for line in text.splitlines(): + for path in _CHECKPOINT_PATH_RE.findall(line): + fragment = _SHA_FRAGMENT_RE.search(line) + full = _SHA_FULL_RE.search(line) + committed = None + if full: + committed = full.group(0) + elif fragment: + committed = f"{fragment.group(1)}…{fragment.group(2)}" + rows.append({"path": path, "committed_sha": committed}) + return rows + + +def _iter_scoreboards() -> list[tuple[Path, dict[str, Any]]]: + """Local eval_*.json scoreboards (path, payload).""" + out: list[tuple[Path, dict[str, Any]]] = [] + for path in sorted(RUNS_ROOT.glob("*/eval*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception: # noqa: BLE001 - unreadable artifact is skipped, noted + continue + if isinstance(payload, dict) and ( + "decode_timeout_count" in payload or "fallback_count" in payload + ): + out.append((path, payload)) + return out + + +def _design_scoreboards() -> list[tuple[Path, dict[str, Any]]]: + """Committed docs/design iter JSONs carrying scoreboard-shaped dicts.""" + out: list[tuple[Path, dict[str, Any]]] = [] + + def _walk(node: Any, path: Path) -> None: + if isinstance(node, dict): + if "decode_timeout_count" in node and "fallback_count" in node: + out.append((path, node)) + return + for value in node.values(): + _walk(value, path) + elif isinstance(node, list): + for value in node: + _walk(value, path) + + for path in sorted(DESIGN_DIR.glob("iter*.json")): + try: + _walk(json.loads(path.read_text(encoding="utf-8")), path) + except Exception: # noqa: BLE001 + continue + return out + + +def run_census(*, out_path: Path = CENSUS_JSON) -> dict[str, Any]: + roster = _roster_rows(MODEL_CARD.read_text(encoding="utf-8")) + roster_by_path = {row["path"]: row for row in roster} + + local_checkpoints = sorted(RUNS_ROOT.glob("*/checkpoints/*.pt")) + checkpoints: list[dict[str, Any]] = [] + seen: set[str] = set() + for path in local_checkpoints: + rel = str(path) + seen.add(rel) + actual = sha256_file(path) + committed = (roster_by_path.get(rel) or {}).get("committed_sha") + checkpoints.append( + { + "path": rel, + "exists_locally": True, + "in_roster": rel in roster_by_path, + "committed_sha": committed, + "actual_sha256": actual, + "committed_sha_match": sha_matches_committed(committed, actual), + } + ) + for rel, row in sorted(roster_by_path.items()): + if rel not in seen: + checkpoints.append( + { + "path": rel, + "exists_locally": False, + "in_roster": True, + "committed_sha": row["committed_sha"], + "actual_sha256": None, + "committed_sha_match": None, + } + ) + + scoreboards: list[dict[str, Any]] = [] + for path, payload in _iter_scoreboards(): + entry = { + "scoreboard_path": str(path), + "source": "local_outputs", + "checkpoint": payload.get("checkpoint"), + "checkpoint_sha256": payload.get("checkpoint_sha256"), + "suite": payload.get("suite"), + **classify_scoreboard(payload), + } + scoreboards.append(entry) + for path, node in _design_scoreboards(): + scoreboards.append( + { + "scoreboard_path": str(path), + "source": "docs_design_historical", + "checkpoint": node.get("checkpoint"), + "checkpoint_sha256": node.get("checkpoint_sha256"), + "suite": node.get("suite"), + **classify_scoreboard(node), + } + ) + + hash_pins: dict[str, Any] = {} + for key, spec in SWEEP_CHECKPOINTS.items(): + path = Path(spec["path"]) + pin: dict[str, Any] = {"path": spec["path"], "exists_locally": path.exists()} + if path.exists(): + actual = sha256_file(path) + pin["actual_sha256"] = actual + recorded = None + sb_path = Path(spec["scoreboard"]) + if sb_path.exists(): + recorded = json.loads(sb_path.read_text(encoding="utf-8")).get( + "checkpoint_sha256" + ) + pin["scoreboard_recorded_sha256"] = recorded + pin["matches_scoreboard_record"] = ( + None if recorded is None else actual == recorded + ) + hash_pins[key] = pin + + census = { + "schema": "slm303_decode_census/v1", + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "taxonomy": list(DECODE_OUTCOMES), + "checkpoints": checkpoints, + "scoreboards": scoreboards, + "hash_pins": hash_pins, + "summary": { + "n_local_checkpoints": len(local_checkpoints), + "n_roster_rows": len(roster), + "n_committed_sha_verified": sum( + 1 for c in checkpoints if c["committed_sha_match"] is True + ), + "n_committed_sha_mismatch": sum( + 1 for c in checkpoints if c["committed_sha_match"] is False + ), + "n_unverifiable": sum( + 1 for c in checkpoints if c["committed_sha_match"] is None + ), + "scoreboard_classes": dict( + sorted( + { + cls: sum( + 1 for s in scoreboards if s["classification"] == cls + ) + for cls in { + s["classification"] for s in scoreboards + } + }.items() + ) + ), + }, + } + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(census, indent=2) + "\n", encoding="utf-8") + return census + + +# --------------------------------------------------------------------------- +# Sweep cells (hard-killed worker per (checkpoint, arm, record)) +# --------------------------------------------------------------------------- + + +def _load_smoke_records() -> dict[str, Any]: + from slm_training.dsl.schema import ExampleRecord + + records: dict[str, Any] = {} + with TEST_SEEDS.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + row = json.loads(line) + if row["id"] in SMOKE_RECORD_IDS: + records[row["id"]] = ExampleRecord( + id=row["id"], + prompt=row["prompt"], + openui=row["openui"], + placeholders=list(row.get("placeholders", [])), + split=row.get("split", "smoke"), + source=row.get("source", "smoke"), + ) + return records + + +def _cell_worker( + checkpoint_path: str, arm_budget: dict[str, Any], record_id: str, queue: Any +) -> None: + """Load the checkpoint fresh, decode one smoke record under the arm budget.""" + try: + from slm_training.harnesses.model_build.plugin import GenerationRequest + from slm_training.models.twotower import TwoTowerModel + + record = _load_smoke_records()[record_id] + model = TwoTowerModel.from_checkpoint(checkpoint_path, device="cpu") + model.config.grammar_ltr_max_tokens = int(arm_budget["canvas_cap"]) + request = GenerationRequest.from_record(record) + t0 = time.perf_counter() + pred = model.generate_batch_requests( + [request], max_len=int(arm_budget["canvas_cap"]) + )[0] + queue.put( + { + "status": "ok", + "elapsed_ms": round((time.perf_counter() - t0) * 1000.0, 2), + "prediction": pred, + } + ) + except Exception as exc: # noqa: BLE001 - decode failure is data + queue.put({"status": "harness_error", "error": str(exc)[:400]}) + + +def run_sweep_cell( + *, + checkpoint_key: str, + arm: str, + record_id: str, + out_dir: Path = CELLS_DIR, +) -> dict[str, Any]: + import multiprocessing as mp + + spec = SWEEP_CHECKPOINTS[checkpoint_key] + arms = build_sweep_arms(int(spec["recorded_canvas_cap"])) + budget = arms[arm] + checkpoint_path = spec["path"] + checkpoint_sha = sha256_file(Path(checkpoint_path)) + record = _load_smoke_records()[record_id] + record_sha = hashlib.sha256( + json.dumps(record.to_dict(), sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + ctx = mp.get_context("fork") + queue: Any = ctx.Queue() + proc = ctx.Process( + target=_cell_worker, args=(checkpoint_path, budget, record_id, queue) + ) + t0 = time.perf_counter() + proc.start() + proc.join(float(budget["timeout_s"])) + elapsed_ms = round((time.perf_counter() - t0) * 1000.0, 2) + if proc.is_alive(): + proc.terminate() + proc.join(30) + outcome = classify_decode_outcome( + parse_ok=None, timed_out=True, fallback_counters=0 + ) + cell = { + "status": "decode_timeout", + "outcome": outcome, + "stop_reason": "decode_timeout", + "elapsed_ms": elapsed_ms, + "prediction": None, + "parse_ok": None, + } + else: + result = ( + queue.get() + if not queue.empty() + else { + "status": "harness_error", + "error": f"worker exited without result (code {proc.exitcode})", + } + ) + if result["status"] == "ok": + pred = str(result["prediction"]) + from slm_training.dsl.parser import validate + + try: + validate(pred) + parse_ok: bool | None = True + error = None + except Exception as exc: # noqa: BLE001 - parse fact, never raised + parse_ok = False + error = str(exc)[:200] + outcome = classify_decode_outcome( + parse_ok=parse_ok, + error=error, + abstained=not pred.strip(), + fallback_counters=0, + timed_out=False, + ) + cell = { + "status": "ok", + "outcome": outcome, + "stop_reason": "empty_prediction" if not pred.strip() else "completed", + "elapsed_ms": result["elapsed_ms"], + "prediction": pred, + "parse_ok": parse_ok, + "error": error, + } + else: + error_text = str(result.get("error") or "") + not_rerunnable = "output contract" in error_text + cell = { + "status": "not_rerunnable" if not_rerunnable else "harness_error", + # No decode attempt happened → no outcome, never a guess. + "outcome": None if not_rerunnable else classify_decode_outcome( + parse_ok=None, harness_exception=True + ), + "stop_reason": "not_rerunnable" if not_rerunnable else "harness_error", + "elapsed_ms": elapsed_ms, + "prediction": None, + "parse_ok": None, + "error": result.get("error"), + } + + record_out = ( + DecodeOutcomeRecordV1( + request_id=record_id, + outcome=cell["outcome"], + stop_reason=cell["stop_reason"], + budget={ + "timeout_s": budget["timeout_s"], + "canvas_cap": budget["canvas_cap"], + "gen_steps": None, + "max_attempts": 1, + }, + elapsed_ms=cell["elapsed_ms"], + fallback_used=False, + detail=cell.get("error"), + ) + if cell["outcome"] is not None + else None + ) + budget_dict = { + "timeout_s": budget["timeout_s"], + "canvas_cap": budget["canvas_cap"], + "gen_steps": None, + "max_attempts": 1, + } + cell.update( + { + "schema": "slm303_sweep_cell/v1", + "checkpoint": checkpoint_key, + "checkpoint_path": checkpoint_path, + "checkpoint_sha256": checkpoint_sha, + "arm": arm, + "budget": budget_dict, + "record_id": record_id, + "record_sha256": record_sha, + "seed": 0, + "outcome_record": record_out.to_dict() if record_out else None, + } + ) + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / f"{checkpoint_key}__{arm}__{record_id}.json" + path.write_text(json.dumps(cell, indent=2) + "\n", encoding="utf-8") + return cell + + +# --------------------------------------------------------------------------- +# Report (census + sweep cells → disposition + durable docs) +# --------------------------------------------------------------------------- + + +def _predeclaration() -> dict[str, Any]: + return { + "experiment_id": "SLM-303", + "hypothesis": ( + "A material share of historical zero/interference-flagged " + "scoreboards are runtime/harness budget artifacts (decode timeout " + "or fallback), not model behavior; widening the decode budget 10x " + "flips timeout-classified rows to model outcomes." + ), + "falsifier": ( + "10x budget does not change any timeout-classified record's " + "outcome on the hash-pinned checkpoints, or the census shows the " + "flagged scoreboards were already clean model behavior." + ), + "arms": { + "baseline": "recorded canvas cap (256/128) + inferred 10s timeout", + "budget10x": "10x timeout and 10x canvas cap; nothing else varies", + }, + "budget_evidence": BASELINE_TIMEOUT_EVIDENCE, + "records": { + "ids": list(SMOKE_RECORD_IDS), + "source": str(TEST_SEEDS), + "disclosure": ( + "the original eval suite directory is empty; the sweep " + "rebuilds the same 3 smoke records (ids match the recorded " + "scoreboard details) from test_seeds.jsonl" + ), + }, + "seed": 0, + "fixed": ( + "same checkpoint bytes (hash-pinned), same records, same seed, " + "checkpoint-declared decode policy; arms differ only in budget" + ), + "checkpoints": { + key: { + "path": spec["path"], + "recorded_canvas_cap": spec["recorded_canvas_cap"], + "recorded_decode_timeout_count": spec["recorded_decode_timeout_count"], + "selection": "only checkpoints with ACTUAL nonzero-timeout scoreboards", + } + for key, spec in SWEEP_CHECKPOINTS.items() + }, + "excluded": { + "slm230_bounded_recursive_r4_r2": ( + "zero-timeout scoreboard; budget interference not implicated — not needed" + ), + "e173/slm294 and all other local checkpoints": ( + "no committed SHA and no hash-pinned timeout scoreboard → not " + "hash-verifiable for this audit (slm294 additionally " + "feasibility-excluded: >280s/16 tokens constrained decode on rico)" + ), + }, + "claim_class": "diagnostic", + } + + +def _disposition( + census: dict[str, Any], pairs: list[dict[str, Any]] +) -> dict[str, Any]: + timeout_boards = [ + s + for s in census["scoreboards"] + if s["classification"] == "runtime_timeout_interference" + ] + fallback_boards = [ + s + for s in census["scoreboards"] + if s["classification"] == "fallback_interference" + ] + unmeasured_boards = [ + s for s in census["scoreboards"] if s["classification"] == "unmeasured" + ] + flips = [ + p + for p in pairs + if p.get("status") == "paired" + and p.get("outcome_baseline") == "runtime_timeout" + and p.get("outcome_10x") != "runtime_timeout" + ] + not_rerunnable = [p for p in pairs if p.get("outcome_baseline") is None] + annotations = [] + for board in timeout_boards: + annotations.append( + { + "scoreboard": board["scoreboard_path"], + "annotation": ( + f"runtime/harness artifact: {board['decode_timeout_count']}" + f"/{board['n']} decode timeouts at canvas cap " + f"{board['decode_canvas_cap']}; zero parse rate on these " + "rows is budget interference, NOT a model verdict" + ), + } + ) + for board in fallback_boards: + annotations.append( + { + "scoreboard": board["scoreboard_path"], + "annotation": ( + f"fallback interference: fallback_count={board['fallback_count']}; " + "fallback outputs never classify as model success" + ), + } + ) + if not pairs: + sweep_verdict = "sweep cells not run" + elif not_rerunnable: + sweep_verdict = ( + f"not_rerunnable: {len(not_rerunnable)}/{len(pairs)} paired cells " + "blocked by the output-contract gate (checkpoint v0 vs required " + "symbol_only/v2, no migration path); the 10x-budget flip question " + "is UNANSWERED by re-decode — census + taxonomy carry the audit" + ) + else: + sweep_verdict = ( + f"{len(flips)} timeout-classified rows flipped outcome under 10x budget" + ) + recommendation = { + "powered_rerun": ( + "at most one: retrain the remediated recipe from symbol-only " + "targets (output contract v2 — the v0 checkpoints are blocked by " + "require_current_output_contract with no migration path), then a " + "preregistered full-smoke re-eval at recorded vs 10x decode " + "budget with per-record decode_outcome fields (now emitted by " + "eval_runner), n≥16 smoke+fixture records for Wilson resolution" + ), + "rationale": ( + "census localizes all timeout interference to the two remediated " + "checkpoints; no other hash-verifiable checkpoint carries a " + "timeout-flagged scoreboard, and re-decode of the v0 checkpoints " + "under current code is impossible" + ), + } + return { + "annotations": annotations, + "counts": { + "runtime_timeout_interference": len(timeout_boards), + "fallback_interference": len(fallback_boards), + "unmeasured": len(unmeasured_boards), + }, + "sweep_verdict": sweep_verdict, + "recommendation": recommendation, + } + + +def render_markdown(payload: dict[str, Any]) -> str: + summary = payload["census"]["summary"] + lines = [ + "# SLM-303 (LAR1-05): decode-budget audit — census, 10x sweep, disposition", + "", + f"- generated_at: `{payload['generated_at']}`", + f"- claim_class: `{payload['predeclaration']['claim_class']}` (diagnostic; not ship evidence)", + f"- taxonomy: `{', '.join(payload['census']['taxonomy'])}`", + f"- budget evidence: {payload['predeclaration']['budget_evidence']}", + "", + "## Census", + "", + f"- local checkpoints enumerated: {summary['n_local_checkpoints']} " + f"(roster rows: {summary['n_roster_rows']})", + f"- committed SHA verified: {summary['n_committed_sha_verified']}; " + f"mismatch: {summary['n_committed_sha_mismatch']}; " + f"unverifiable (no committed SHA): {summary['n_unverifiable']}", + f"- scoreboard classes: `{json.dumps(summary['scoreboard_classes'])}`", + "", + "### Hash pins (sweep checkpoints)", + "", + "| key | path | sha256 | matches scoreboard record |", + "| --- | --- | --- | --- |", + ] + for key, pin in payload["census"]["hash_pins"].items(): + lines.append( + f"| {key} | `{pin['path']}` | `{pin.get('actual_sha256')}` | " + f"{pin.get('matches_scoreboard_record')} |" + ) + lines += ["", "## Budget sweep (preregistered, paired)", ""] + sweep = payload.get("sweep") or {} + if sweep.get("pairs"): + lines += [ + "| checkpoint | record | baseline outcome | 10x outcome | baseline ms | 10x ms |", + "| --- | --- | --- | --- | --- | --- |", + ] + for pair in sweep["pairs"]: + if pair.get("status") != "paired": + lines.append( + f"| {pair['checkpoint']} | {pair['record_id']} | unpaired | | | |" + ) + continue + lines.append( + f"| {pair['checkpoint']} | {pair['record_id']} | " + f"{pair['outcome_baseline'] or pair.get('cell_status_baseline')} | " + f"{pair['outcome_10x'] or pair.get('cell_status_10x')} | " + f"{pair['elapsed_ms_baseline']} | {pair['elapsed_ms_10x']} |" + ) + lines += [ + "", + f"- outcome counts (baseline): `{json.dumps(sweep['outcome_counts']['baseline'])}`", + f"- outcome counts (10x): `{json.dumps(sweep['outcome_counts']['budget10x'])}`", + ] + else: + lines.append("- sweep cells not present; see census + disposition only") + lines += ["", "## Disposition (append-only annotations)", ""] + disp = payload["disposition"] + for note in disp["annotations"]: + lines.append(f"- `{note['scoreboard']}`: {note['annotation']}") + lines += [ + "", + f"- sweep verdict: {disp['sweep_verdict']}", + f"- powered-rerun recommendation: {disp['recommendation']['powered_rerun']}", + f" - rationale: {disp['recommendation']['rationale']}", + "", + "Historical iter docs and MODEL_CARD rows are untouched; these " + "annotations are additive.", + "", + ] + return "\n".join(lines) + + +def run_report( + *, + json_out: Path = DEFAULT_JSON_OUT, + md_out: Path = DEFAULT_MD_OUT, +) -> dict[str, Any]: + census = run_census() + cells: list[dict[str, Any]] = [] + if CELLS_DIR.exists(): + for path in sorted(CELLS_DIR.glob("*.json")): + cells.append(json.loads(path.read_text(encoding="utf-8"))) + pairs = pair_cells(cells) + outcomes_by_arm = { + arm: outcome_counts( + [c["outcome"] for c in cells if c["arm"] == arm and c["outcome"]] + ) + for arm in ("baseline", "budget10x") + } + payload: dict[str, Any] = { + "schema": "slm303_decode_budget_audit/v1", + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "predeclaration": _predeclaration(), + "census": census, + "sweep": { + "n_cells": len(cells), + "pairs": pairs, + "outcome_counts": outcomes_by_arm, + "cells": cells, + }, + "disposition": _disposition(census, pairs), + } + payload["version_stamp"] = build_version_stamp(COMPONENT, EVAL_COMPONENT) + json_out.write_text( + json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8" + ) + md_out.write_text(render_markdown(payload), encoding="utf-8") + return payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="mode", required=True) + sub.add_parser("census", help="checkpoint/scoreboard census + hash pins") + cell = sub.add_parser("sweep-cell", help="run one (checkpoint, arm, record) cell") + cell.add_argument("--checkpoint", required=True, choices=sorted(SWEEP_CHECKPOINTS)) + cell.add_argument("--arm", required=True, choices=("baseline", "budget10x")) + cell.add_argument("--record-id", required=True, choices=list(SMOKE_RECORD_IDS)) + rep = sub.add_parser("report", help="merge census + cells into durable docs") + rep.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + rep.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + args = parser.parse_args(argv) + + if args.mode == "census": + census = run_census() + print(json.dumps(census["summary"], indent=2)) + print(f"wrote {CENSUS_JSON}") + return 0 + if args.mode == "sweep-cell": + result = run_sweep_cell( + checkpoint_key=args.checkpoint, + arm=args.arm, + record_id=args.record_id, + ) + print( + f"{args.checkpoint}/{args.arm}/{args.record_id}: " + f"{result['outcome']} ({result['status']}, {result['elapsed_ms']}ms)" + ) + return 0 + payload = run_report(json_out=args.json_out, md_out=args.md_out) + print(f"sweep verdict: {payload['disposition']['sweep_verdict']}") + print(f"wrote {args.json_out} and {args.md_out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/model_build/decode_outcome.py b/src/slm_training/harnesses/model_build/decode_outcome.py new file mode 100644 index 000000000..b447aac44 --- /dev/null +++ b/src/slm_training/harnesses/model_build/decode_outcome.py @@ -0,0 +1,140 @@ +"""SLM-303 (LAR1-05): decode-outcome taxonomy for budget/interference audits. + +Every decode attempt lands in exactly one of ``DECODE_OUTCOMES``. The point of +the taxonomy is to stop laundering runtime/harness artifacts (timeouts, +fallbacks, harness exceptions) into model-quality claims: a fallback output is +never a model success, and a timeout says something about the budget, not the +model. + +Classification precedence is strict and total: + + harness_error > runtime_timeout > fallback_output > model_abstain + > model_invalid / model_valid +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any, Mapping + +DECODE_OUTCOMES: tuple[str, ...] = ( + "model_valid", + "model_invalid", + "model_abstain", + "runtime_timeout", + "fallback_output", + "harness_error", +) + +MODEL_VALID, MODEL_INVALID, MODEL_ABSTAIN = DECODE_OUTCOMES[0:3] +RUNTIME_TIMEOUT, FALLBACK_OUTPUT, HARNESS_ERROR = DECODE_OUTCOMES[3:6] + +# DecodeStats counter names that mean "the emitted output came from a +# non-model fallback path". Mirrors the eval_runner fallback_count set. +FALLBACK_COUNTER_NAMES: tuple[str, ...] = ( + "unconstrained_retries", + "compiler_fallbacks", + "seeded_fallbacks", + "template_fallback_count", + "certified_fallbacks", +) + + +def fallback_counter_total(stats: object | None) -> int: + """Total fallback counter value across a DecodeStats-like row (0 if none).""" + if stats is None: + return 0 + if isinstance(stats, Mapping): + return sum(int(stats.get(name) or 0) for name in FALLBACK_COUNTER_NAMES) + return sum(int(getattr(stats, name, 0) or 0) for name in FALLBACK_COUNTER_NAMES) + + +def classify_decode_outcome( + *, + parse_ok: bool | None, + error: str | None = None, + fallback_counters: int = 0, + timed_out: bool = False, + abstained: bool = False, + harness_exception: bool = False, +) -> str: + """Classify one decode attempt into the taxonomy. + + Precedence is strict: a harness exception dominates everything; a timeout + is a budget artifact, not a model verdict; a fallback output is never a + model success even when it happens to parse; an abstention (empty output + without timeout/fallback) outranks a parse verdict. + """ + if harness_exception: + return HARNESS_ERROR + if timed_out: + return RUNTIME_TIMEOUT + if fallback_counters > 0: + return FALLBACK_OUTPUT + if abstained: + return MODEL_ABSTAIN + # parse_ok None = parse not evaluated; a produced candidate with no + # contrary evidence is a model output, not a failure verdict. + return MODEL_INVALID if parse_ok is False else MODEL_VALID + + +@dataclass +class DecodeOutcomeRecordV1: + """One decode attempt with its budget, evidence, and classified outcome.""" + + request_id: str + outcome: str + stop_reason: str + budget: dict[str, Any] = field(default_factory=dict) + elapsed_ms: float | None = None + forwards: int | None = None + verifier_calls: int | None = None + fallback_used: bool = False + detail: str | None = None + + schema: str = "decode_outcome_record/v1" + + def __post_init__(self) -> None: + if self.outcome not in DECODE_OUTCOMES: + raise ValueError(f"unknown decode outcome {self.outcome!r}") + if self.outcome == FALLBACK_OUTPUT and not self.fallback_used: + raise ValueError("fallback_output outcome requires fallback_used=True") + if self.outcome == MODEL_VALID and self.fallback_used: + raise ValueError("a fallback output may never classify as model_valid") + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "DecodeOutcomeRecordV1": + return cls( + request_id=str(data["request_id"]), + outcome=str(data["outcome"]), + stop_reason=str(data["stop_reason"]), + budget=dict(data.get("budget") or {}), + elapsed_ms=data.get("elapsed_ms"), + forwards=data.get("forwards"), + verifier_calls=data.get("verifier_calls"), + fallback_used=bool(data.get("fallback_used", False)), + detail=data.get("detail"), + ) + + +def outcome_counts(outcomes: list[str]) -> dict[str, int]: + """Per-taxonomy counts with every key present (zeros included).""" + counts = {name: 0 for name in DECODE_OUTCOMES} + for outcome in outcomes: + if outcome not in counts: + raise ValueError(f"unknown decode outcome {outcome!r}") + counts[outcome] += 1 + return counts + + +__all__ = [ + "DECODE_OUTCOMES", + "FALLBACK_COUNTER_NAMES", + "DecodeOutcomeRecordV1", + "classify_decode_outcome", + "fallback_counter_total", + "outcome_counts", +] diff --git a/src/slm_training/harnesses/model_build/eval_runner.py b/src/slm_training/harnesses/model_build/eval_runner.py index 6d1524b12..645808776 100644 --- a/src/slm_training/harnesses/model_build/eval_runner.py +++ b/src/slm_training/harnesses/model_build/eval_runner.py @@ -44,6 +44,7 @@ DEFAULT_SHIP_GATES, ) from slm_training.models.decode_stats import collect_decode_stats +from slm_training.harnesses.model_build.decode_outcome import outcome_counts from slm_training.versioning import component_version _COMPONENT_RE = re.compile(r"\b([A-Z][A-Za-z0-9]*)\s*\(") @@ -970,6 +971,11 @@ def _generate_chunk_unbounded( return out, list(evidence) decode_timeout_count = 0 + # SLM-303: per-chunk decode-outcome evidence (parallel to the chunk loop); + # each entry carries the timeout fact plus the chunk's DecodeStats row (or + # None when the plugin exposes no stats) so every scored record can be + # classified into the decode-outcome taxonomy. + chunk_decode_meta: list[dict[str, Any]] = [] def _generate_chunk( chunk: list[ExampleRecord], @@ -977,8 +983,18 @@ def _generate_chunk( """Generate a chunk, converting an explicit diagnostic timeout to failures.""" nonlocal decode_timeout_count seconds = float(getattr(config, "decode_timeout_seconds", 0) or 0) + + def _run(timed_out: bool) -> tuple[list[str], list[dict[str, Any]]]: + stats_before = len(decode_stats_rows) + result = _generate_chunk_unbounded(chunk) + stats = ( + decode_stats_rows[-1] if len(decode_stats_rows) > stats_before else None + ) + chunk_decode_meta.append({"timed_out": timed_out, "stats": stats}) + return result + if seconds <= 0 or not hasattr(signal, "setitimer"): - return _generate_chunk_unbounded(chunk) + return _run(timed_out=False) def _alarm(_signum: int, _frame: object) -> None: raise TimeoutError(f"decode exceeded {seconds:g}s") @@ -986,23 +1002,80 @@ def _alarm(_signum: int, _frame: object) -> None: previous = signal.signal(signal.SIGALRM, _alarm) signal.setitimer(signal.ITIMER_REAL, seconds) try: - return _generate_chunk_unbounded(chunk) + return _run(timed_out=False) except TimeoutError as exc: stats = getattr(exc, "decode_stats", None) if stats is not None: _annotate_decode_trace_records(stats, chunk) decode_stats_rows.append(stats) + chunk_decode_meta.append({"timed_out": True, "stats": stats}) decode_timeout_count += len(chunk) return ["" for _ in chunk], [] finally: signal.setitimer(signal.ITIMER_REAL, 0) signal.signal(signal.SIGALRM, previous) + def _decode_outcome_fields( + pred: str, + *, + parse_ok: bool | None, + error: str | None, + decode_meta: dict[str, Any] | None, + ) -> dict[str, Any]: + """SLM-303 additive per-record decode-outcome classification. + + Computed from the same evidence the runner already has (chunk timeout + fact, chunk DecodeStats fallback counters, parse verdict); never + changes any existing field. + """ + from slm_training.harnesses.model_build.decode_outcome import ( + MODEL_VALID, + classify_decode_outcome, + fallback_counter_total, + ) + + meta = decode_meta or {} + stats = meta.get("stats") + fallbacks = fallback_counter_total(stats) + timed_out = bool(meta.get("timed_out")) + outcome = classify_decode_outcome( + parse_ok=parse_ok, + error=error, + fallback_counters=fallbacks, + timed_out=timed_out, + abstained=not pred.strip(), + harness_exception=False, + ) + if timed_out: + stop_reason = "decode_timeout" + else: + stop_reason = "" + for name in ( + "compiler_lattice_termination_reason", + "solver_terminal_status", + ): + value = str(getattr(stats, name, "") or "") if stats is not None else "" + if value: + stop_reason = value + break + if not stop_reason: + stop_reason = "empty_prediction" if not pred.strip() else "completed" + detail = None + if outcome == MODEL_VALID and parse_ok is None: + detail = "parse_not_evaluated" + return { + "decode_outcome": outcome, + "stop_reason": stop_reason, + "fallback_used": fallbacks > 0, + "decode_outcome_detail": detail, + } + def _score_one( record: ExampleRecord, pred: str, latency_ms: float, prediction_evidence: dict[str, Any] | None = None, + decode_meta: dict[str, Any] | None = None, ) -> None: nonlocal parse_ok, syntax_parse_ok, raw_syntax_ok nonlocal match_error_count, reward_error_count, empty_prediction_count @@ -1036,6 +1109,9 @@ def _score_one( ).encode("utf-8") ).hexdigest(), "topology_evidence": evidence or None, + **_decode_outcome_fields( + pred, parse_ok=None, error=None, decode_meta=decode_meta + ), } ) task_cases.append( @@ -1169,6 +1245,9 @@ def _score_one( ).hexdigest(), "serialized": serialized, "topology_evidence": evidence or None, + **_decode_outcome_fields( + pred, parse_ok=ok, error=error, decode_meta=decode_meta + ), } ) task_cases.append( @@ -1194,16 +1273,18 @@ def _score_one( chunk = records[start : start + batch_size] t0 = time.perf_counter() preds, evidence_rows = _generate_chunk(chunk) + chunk_meta = chunk_decode_meta[-1] if chunk_decode_meta else None elapsed = (time.perf_counter() - t0) * 1000.0 per = elapsed / max(1, len(chunk)) for index, (record, pred) in enumerate(zip(chunk, preds)): latencies.append(per) evidence = evidence_rows[index] if index < len(evidence_rows) else None - _score_one(record, pred, per, evidence) + _score_one(record, pred, per, evidence, chunk_meta) else: for record in records: t0 = time.perf_counter() predictions, evidence_rows = _generate_chunk([record]) + chunk_meta = chunk_decode_meta[-1] if chunk_decode_meta else None pred = predictions[0] latencies.append((time.perf_counter() - t0) * 1000.0) _score_one( @@ -1211,6 +1292,7 @@ def _score_one( pred, latencies[-1], evidence_rows[0] if evidence_rows else None, + chunk_meta, ) lat_sorted = sorted(latencies) @@ -1392,6 +1474,11 @@ def _rate_evidence(successes: int, total: int) -> dict[str, Any]: "failure_breakdown": failure_breakdown, "decode_timeout_count": decode_timeout_count, "decode_canvas_cap": canvas_cap, + # SLM-303: per-taxonomy decode-outcome counts over details[] rows + # (additive; every taxonomy key always present). + "decode_outcome_counts": outcome_counts( + [str(row.get("decode_outcome")) for row in details] + ), "details": details, "generation_evidence_schemas": sorted( { diff --git a/src/slm_training/harnesses/model_build/ship_gates.py b/src/slm_training/harnesses/model_build/ship_gates.py index 9b9673e77..953f8f203 100644 --- a/src/slm_training/harnesses/model_build/ship_gates.py +++ b/src/slm_training/harnesses/model_build/ship_gates.py @@ -66,7 +66,7 @@ # (candidate_pending_calibration) so recording it can never green a gate. MEANINGFUL_METRIC_POLICY = { "active_primary": "meaningful_program_v1", - "threshold_version": "openui_ship_gates_v2", + "threshold_version": "openui_ship_gates_v3", "meaningful_program_v1": { "version": "1.0.0", "wire_field": "meaningful_program_rate", diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 40e8c67b4..45fadec38 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -4521,8 +4521,26 @@ } ] }, + "harness.experiments.slm303_decode_budget_audit": { + "version": "v1", + "kind": "harness", + "paths": [ + "scripts/run_slm303_decode_budget_audit.py", + "tests/test_harnesses/model_build/test_decode_outcome.py", + "tests/test_scripts/test_run_slm303_decode_budget_audit.py", + "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "docs/design/iter-slm303-decode-budget-audit-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-303 decode-budget audit with hash pins, preregistered budget sweep, and taxonomy disposition" + } + ] + }, "harness.model_build.eval": { - "version": "v52", + "version": "v53", "kind": "harness", "paths": [ "src/slm_training/harnesses/model_build/debt_router.py", @@ -4533,6 +4551,11 @@ "scripts/evaluate_model.py" ], "history": [ + { + "version": "v53", + "date": "2026-07-24", + "note": "Add SLM-303 per-record decode outcome taxonomy and suite outcome counts without changing existing gate inputs" + }, { "version": "v52", "date": "2026-07-24", diff --git a/tests/test_harnesses/model_build/test_decode_outcome.py b/tests/test_harnesses/model_build/test_decode_outcome.py new file mode 100644 index 000000000..7020f8001 --- /dev/null +++ b/tests/test_harnesses/model_build/test_decode_outcome.py @@ -0,0 +1,115 @@ +"""SLM-303: decode-outcome taxonomy tests.""" + +from __future__ import annotations + +import pytest + +from slm_training.harnesses.model_build.decode_outcome import ( + DECODE_OUTCOMES, + DecodeOutcomeRecordV1, + classify_decode_outcome, + fallback_counter_total, + outcome_counts, +) + + +def test_taxonomy_has_six_outcomes() -> None: + assert DECODE_OUTCOMES == ( + "model_valid", + "model_invalid", + "model_abstain", + "runtime_timeout", + "fallback_output", + "harness_error", + ) + + +def test_precedence_harness_error_dominates() -> None: + assert ( + classify_decode_outcome( + parse_ok=True, + fallback_counters=3, + timed_out=True, + abstained=True, + harness_exception=True, + ) + == "harness_error" + ) + + +def test_precedence_timeout_beats_fallback_and_parse() -> None: + assert ( + classify_decode_outcome( + parse_ok=True, fallback_counters=2, timed_out=True + ) + == "runtime_timeout" + ) + + +def test_fallback_never_model_success() -> None: + # Even a parseable output is a fallback artifact when a fallback fired. + assert ( + classify_decode_outcome(parse_ok=True, fallback_counters=1) + == "fallback_output" + ) + assert ( + classify_decode_outcome(parse_ok=False, fallback_counters=1) + == "fallback_output" + ) + + +def test_abstain_beats_parse_verdict() -> None: + assert classify_decode_outcome(parse_ok=False, abstained=True) == "model_abstain" + + +def test_model_valid_and_invalid() -> None: + assert classify_decode_outcome(parse_ok=True) == "model_valid" + assert classify_decode_outcome(parse_ok=False) == "model_invalid" + # parse not evaluated + produced output → model_valid with no parse claim + assert classify_decode_outcome(parse_ok=None) == "model_valid" + + +def test_fallback_counter_total_mapping_and_object() -> None: + assert fallback_counter_total(None) == 0 + assert fallback_counter_total({"compiler_fallbacks": 2, "seeded_fallbacks": 1}) == 3 + + class _Stats: + unconstrained_retries = 1 + template_fallback_count = 2 + + assert fallback_counter_total(_Stats()) == 3 + + +def test_outcome_counts_all_keys_present() -> None: + counts = outcome_counts(["model_valid", "model_valid", "runtime_timeout"]) + assert counts["model_valid"] == 2 + assert counts["runtime_timeout"] == 1 + assert counts["harness_error"] == 0 + assert set(counts) == set(DECODE_OUTCOMES) + with pytest.raises(ValueError): + outcome_counts(["not_an_outcome"]) + + +def test_record_roundtrip_and_guards() -> None: + record = DecodeOutcomeRecordV1( + request_id="r1", + outcome="model_valid", + stop_reason="completed", + budget={"timeout_s": 10.0, "canvas_cap": 256}, + elapsed_ms=12.5, + ) + clone = DecodeOutcomeRecordV1.from_dict(record.to_dict()) + assert clone == record + with pytest.raises(ValueError): + DecodeOutcomeRecordV1(request_id="r", outcome="bogus", stop_reason="x") + with pytest.raises(ValueError): + DecodeOutcomeRecordV1( + request_id="r", outcome="fallback_output", stop_reason="x" + ) + with pytest.raises(ValueError): + DecodeOutcomeRecordV1( + request_id="r", + outcome="model_valid", + stop_reason="x", + fallback_used=True, + ) diff --git a/tests/test_scripts/test_run_slm303_decode_budget_audit.py b/tests/test_scripts/test_run_slm303_decode_budget_audit.py new file mode 100644 index 000000000..56b337d53 --- /dev/null +++ b/tests/test_scripts/test_run_slm303_decode_budget_audit.py @@ -0,0 +1,159 @@ +"""SLM-303: census / hash-pin / sweep-pairing tests (pure functions only).""" + +from __future__ import annotations + +import pytest + +from scripts.run_slm303_decode_budget_audit import ( + SWEEP_CHECKPOINTS, + build_sweep_arms, + classify_scoreboard, + pair_cells, + sha_matches_committed, +) + +ACTUAL = "653e71b286964373eda30ff7fb28a836bf0514c6bf78cac9fb07a12223f8fa3a" + + +def test_sha_fragment_prefix_suffix_match() -> None: + assert sha_matches_committed("653e71b2…f8fa3a", ACTUAL) is True + assert sha_matches_committed("`653e71b2…f8fa3a`", ACTUAL) is True + assert sha_matches_committed("deadbeef…f8fa3a", ACTUAL) is False + assert sha_matches_committed("653e71b2…0000", ACTUAL) is False + + +def test_sha_full_match_and_mismatch() -> None: + assert sha_matches_committed(ACTUAL, ACTUAL) is True + assert sha_matches_committed("0" * 64, ACTUAL) is False + + +def test_sha_uncommitted_is_unverifiable_not_pass() -> None: + assert sha_matches_committed(None, ACTUAL) is None + assert sha_matches_committed("", ACTUAL) is None + assert sha_matches_committed("not-a-sha", ACTUAL) is None + + +def _scoreboard(**overrides): + base = { + "n": 3, + "decode_timeout_count": 0, + "fallback_count": 0, + "decode_canvas_cap": 256, + "parse_rate": 0.0, + "details": [], + } + base.update(overrides) + return base + + +def test_classify_scoreboard_unmeasured_when_fallback_null() -> None: + result = classify_scoreboard(_scoreboard(fallback_count=None)) + assert result["classification"] == "unmeasured" + + +def test_classify_scoreboard_timeout_interference() -> None: + result = classify_scoreboard(_scoreboard(decode_timeout_count=3)) + assert result["classification"] == "runtime_timeout_interference" + + +def test_classify_scoreboard_fallback_interference() -> None: + result = classify_scoreboard(_scoreboard(fallback_count=2)) + assert result["classification"] == "fallback_interference" + + +def test_classify_scoreboard_clean_zero_is_model_behavior() -> None: + result = classify_scoreboard(_scoreboard()) + assert result["classification"] == "model_behavior" + + +def test_classify_scoreboard_missing_fields_unmeasured() -> None: + assert classify_scoreboard({"n": 3})["classification"] == "unmeasured" + + +def test_classify_scoreboard_row_outcomes_from_details() -> None: + board = classify_scoreboard( + _scoreboard( + details=[ + {"id": "a", "parse_ok": True, "prediction": "(root)"}, + {"id": "b", "parse_ok": False, "prediction": "junk"}, + {"id": "c", "parse_ok": False, "prediction": ""}, + ] + ) + ) + outcomes = {r["id"]: r["outcome"] for r in board["row_outcomes"]} + assert outcomes == { + "a": "model_valid", + "b": "model_invalid", + "c": "model_abstain", + } + + +def test_sweep_arms_differ_only_in_budget() -> None: + arms = build_sweep_arms(256) + assert set(arms) == {"baseline", "budget10x"} + assert arms["baseline"]["canvas_cap"] == 256 + assert arms["budget10x"]["canvas_cap"] == 2560 + assert arms["budget10x"]["timeout_s"] == 10 * arms["baseline"]["timeout_s"] + + +def _cell(checkpoint: str, arm: str, record: str, outcome: str, sha: str = "abc"): + return { + "checkpoint": checkpoint, + "arm": arm, + "record_id": record, + "outcome": outcome, + "budget": {"timeout_s": 10.0, "canvas_cap": 256}, + "checkpoint_sha256": sha, + "record_sha256": "rec", + "seed": 0, + "elapsed_ms": 5.0, + "parse_ok": None, + } + + +def test_pair_cells_pairs_and_detects_flips() -> None: + cells = [ + _cell("ltr2", "baseline", "smoke_hero_01", "runtime_timeout"), + _cell("ltr2", "budget10x", "smoke_hero_01", "model_invalid"), + ] + (pair,) = pair_cells(cells) + assert pair["status"] == "paired" + assert pair["outcome_baseline"] == "runtime_timeout" + assert pair["outcome_10x"] == "model_invalid" + + +def test_pair_cells_unpaired_and_violation() -> None: + unpaired = pair_cells([_cell("ltr2", "baseline", "smoke_hero_01", "x")]) + assert unpaired[0]["status"] == "unpaired" + violation = pair_cells( + [ + _cell("ltr2", "baseline", "smoke_hero_01", "a", sha="abc"), + _cell("ltr2", "budget10x", "smoke_hero_01", "b", sha="def"), + ] + ) + assert violation[0]["status"] == "pair_violation" + assert "checkpoint_sha256" in violation[0]["pair_violation_fields"] + + +def test_pairing_is_deterministic() -> None: + cells = [ + _cell("ltr2", "budget10x", "smoke_button_01", "model_invalid"), + _cell("ltr2", "baseline", "smoke_hero_01", "runtime_timeout"), + _cell("ltr2", "baseline", "smoke_button_01", "runtime_timeout"), + _cell("ltr2", "budget10x", "smoke_hero_01", "model_invalid"), + ] + assert pair_cells(cells) == pair_cells(list(reversed(cells))) + + +def test_sweep_checkpoints_are_the_two_remediated() -> None: + assert set(SWEEP_CHECKPOINTS) == {"ltr2", "lexer_ltr2"} + for spec in SWEEP_CHECKPOINTS.values(): + assert spec["recorded_decode_timeout_count"] == 3 + assert spec["recorded_canvas_cap"] in (128, 256) + + +@pytest.mark.parametrize("cap", [128, 256]) +def test_arms_track_recorded_cap(cap: int) -> None: + arms = build_sweep_arms(cap) + assert arms["baseline"]["canvas_cap"] == cap + assert arms["budget10x"]["canvas_cap"] == cap * 10 From dbe2fd595f6a40a964e652ebb1a7bca3df6e4d97 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:29:17 -0500 Subject: [PATCH 03/18] SLM-305: extend tree edit language --- README.md | 12 + docs/design/agentv-evaluation.md | 5 + .../iter-slm305-edit-language-20260724.json | 732 ++++++++++++++++++ .../iter-slm305-edit-language-20260724.md | 102 +++ .../langsmith-telemetry-smoke-20260724.json | 51 ++ package-lock.json | 180 ++++- package.json | 4 + pyproject.toml | 4 + scripts/run_agentv_eval.mjs | 55 ++ scripts/run_slm299_reachability_audit.py | 121 ++- src/slm_training/evals/agentv.py | 22 +- .../experiments/slm299_edit_reachability.py | 273 ++++++- .../harnesses/model_build/eval_runner.py | 58 ++ src/slm_training/models/checkpoint_migrate.py | 94 +++ .../models/tree_edit_diffusion.py | 613 ++++++++++++++- src/slm_training/resources/versions.json | 24 +- src/slm_training/runtime/telemetry/trace.py | 199 ++++- tests/test_evals/test_agentv.py | 34 + .../test_slm299_edit_reachability.py | 108 ++- .../model_build/test_eval_gates.py | 39 + tests/test_models/test_tree_edit_diffusion.py | 160 ++++ tests/test_runtime_trace.py | 145 ++++ 22 files changed, 2915 insertions(+), 120 deletions(-) create mode 100644 docs/design/iter-slm305-edit-language-20260724.json create mode 100644 docs/design/iter-slm305-edit-language-20260724.md create mode 100644 docs/design/langsmith-telemetry-smoke-20260724.json diff --git a/README.md b/README.md index 4b01aa277..f0c4cfeeb 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,18 @@ instead of memorizing paths. Selected immutable snapshots publish to Git with Every new run writes `outputs/runs//trace.json` and OTLP JSONL signals under `outputs/traces//`. Set `OTEL_EXPORTER_OTLP_ENDPOINT` for an optional remote OTLP mirror; detailed domain traces remain local and linked by trace ID. +Set `LANGSMITH_TRACING=true` and provide `LANGSMITH_API_KEY` to export an +additional, best-effort aggregate trace to the `slm-training` LangSmith project +(`LANGSMITH_PROJECT`, `LANGSMITH_ENDPOINT`, and `LANGSMITH_WORKSPACE_ID` are +optional overrides). This exports run/suite metrics, version stamps, gate +verdicts, and AgentV summaries only—never prompts, targets, completions, +checkpoints, raw logs, or credentials. Local evidence and AgentEvals remain +authoritative when LangSmith is unavailable. Install the harness-side client +with `pip install -e '.[observability]'`; the shared trace boundary loads the +repository's ignored `.env` without overriding environment variables provided +by CI or a shell. Standard OTLP settings are honored as well: +`OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`, signal-specific endpoints, +and `OTEL_EXPORTER_OTLP_TIMEOUT` (milliseconds; bounded to five seconds). The flush pipeline remains: curated seeds + RICO + Awwwards → deterministic quality synth → per-record DESIGN.md + OpenUI validate → quality gates → stable diff --git a/docs/design/agentv-evaluation.md b/docs/design/agentv-evaluation.md index 1aa131839..12f7a6450 100644 --- a/docs/design/agentv-evaluation.md +++ b/docs/design/agentv-evaluation.md @@ -29,6 +29,10 @@ AgentEvals assertion results has `authority: "AgentEvals assertions"`. under `/evals/`. 5. `write_ship_gates` projects those assertion results into the compatibility `gates.json` shape and records the AgentEvals authority. +6. When opt-in LangSmith tracing is configured, the runner publishes its + aggregate AgentV summary as a child of the existing W3C-correlated run + trace. This is observational only; AgentEvals assertions, local artifacts, + and ship gates retain authority if export fails. The `agentv` npm package is retained for the canonical CLI and dashboard. In the pinned release its published package is CLI-only, so programmatic execution @@ -82,3 +86,4 @@ The implementation check is recorded in | --- | --- | --- | --- | | 2026-07-14 | CPU, steps 0, no model backend; AgentV SDK fixture plus model/loss/task/train-loop harness tests | 34/34 focused tests passed; SDK fixture wrote valid AgentEvals JSONL and AgentV artifacts; dependency audit has 0 high/critical findings | Tooling wiring only; no checkpoint, model score, or ship gate was produced | | 2026-07-23 | CPU, steps 0, no model backend; focused assertion-authority tests with the pinned SDK | AgentEvals JSONL carried required code-graders over raw criteria; the runner produced 1/1 passing fixture criteria with 0 execution errors; 135 focused gate/consumer checks and all 6 interpreted page validations passed. Dashboard production build remained environment-blocked because the locked `@openfeature/web-sdk` dependency was unavailable and automatic review rejected installation before execution. | Tooling and gate-authority wiring only; no checkpoint or model-quality claim | +| 2026-07-24 | Local summary-only telemetry configuration smoke; LangSmith `slm-training` project, two-second bounded flush, standard OTLP resource defaults | W3C root trace `fdf78e5360a6647b09fa0882a2c5d4ec` was published and read back as `slm.telemetry.config` | Tooling wiring only; no checkpoint, model score, or ship gate was produced ([JSON](langsmith-telemetry-smoke-20260724.json)) | diff --git a/docs/design/iter-slm305-edit-language-20260724.json b/docs/design/iter-slm305-edit-language-20260724.json new file mode 100644 index 000000000..6d83d4525 --- /dev/null +++ b/docs/design/iter-slm305-edit-language-20260724.json @@ -0,0 +1,732 @@ +{ + "schema": "slm299_edit_reachability_audit/v2", + "experiment_id": "slm299-edit-reachability", + "seed_source": "root = Stack([], \"column\")", + "mode": "extended", + "max_edits": 8, + "node_budget": 120, + "generated_at": "2026-07-24T20:22:57Z", + "verdict_policy": "reachable_fraction is computed over decided cases only; UNKNOWN_BUDGET cases are reported separately and are never counted as unreachable; suites without a corpus are corpus_unavailable, never zero-reachable. Reachability is a space-coverage proof, never a model-quality claim.", + "suites": { + "train": { + "status": "ok", + "n_cases": 6, + "n_decided": 6, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_direction_change": 3, + "unsupported_component": 3 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "train_auth_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_auth_01_aug_dir", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_auth_01_syn_2", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01_aug_dir", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01_syn_0", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "smoke": { + "status": "ok", + "n_cases": 3, + "n_decided": 3, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 3 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "smoke_hero_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_button_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_callout_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "held_out": { + "status": "ok", + "n_cases": 5, + "n_decided": 5, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 5 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "held_out_form_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_input_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_tabs_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_settings_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "adversarial": { + "status": "ok", + "n_cases": 4, + "n_decided": 4, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_direction_change": 3, + "unsupported_component": 1 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "adv_empty_prompt_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_deep_nest_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_many_buttons_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "ood": { + "status": "ok", + "n_cases": 4, + "n_decided": 4, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 4 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "ood_dashboard_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_gallery_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_modal_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_auth_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "rico": { + "status": "ok", + "n_cases": 6, + "n_decided": 6, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_direction_change": 6 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "rico_eval_test_0", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_1", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_2", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_4", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_8", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_9", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + } + }, + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "5d35b90d7fc58c094604c23d702ab65593f98004", + "code_dirty": true, + "components": { + "harness.experiments.slm299_edit_reachability": "v2" + }, + "stamped_at": "2026-07-24T20:22:57.129778+00:00" + }, + "suites_v1": { + "train": { + "status": "ok", + "n_cases": 6, + "n_decided": 6, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_direction_change": 3, + "unsupported_component": 3 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "train_auth_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_auth_01_aug_dir", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_auth_01_syn_2", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01_aug_dir", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "train_button_row_01_syn_0", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "smoke": { + "status": "ok", + "n_cases": 3, + "n_decided": 3, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_container_add": 2, + "unsupported_component": 1 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "smoke_hero_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_button_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_callout_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "held_out": { + "status": "ok", + "n_cases": 5, + "n_decided": 5, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_container_add": 3, + "unsupported_component": 2 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "held_out_form_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_input_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_tabs_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_settings_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "adversarial": { + "status": "ok", + "n_cases": 4, + "n_decided": 4, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_container_add": 2, + "needs_direction_change": 2 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "adv_empty_prompt_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_deep_nest_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_many_buttons_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "ood": { + "status": "ok", + "n_cases": 4, + "n_decided": 4, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_container_add": 2, + "unsupported_component": 2 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "ood_dashboard_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_gallery_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_modal_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_auth_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "rico": { + "status": "ok", + "n_cases": 6, + "n_decided": 6, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "needs_container_add": 6 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "rico_eval_test_0", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_1", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_2", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_4", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_8", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_9", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_container_add", + "edit_lower_bound": null, + "path_length": 0 + } + ] + } + }, + "old_vs_extended": { + "train": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "smoke": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "held_out": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "adversarial": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "ood": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "rico": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + } + }, + "x22_evidence_annotations": [ + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "adversarial", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided adversarial cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "adversarial", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided adversarial cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "held_out", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "held_out", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "ood", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "ood", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "rico", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided rico cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "rico", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided rico cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "smoke", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "smoke", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "train", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided train cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on train in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-24T20:22:57Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "train", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided train cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on train in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + } + ] +} diff --git a/docs/design/iter-slm305-edit-language-20260724.md b/docs/design/iter-slm305-edit-language-20260724.md new file mode 100644 index 000000000..d9f85ae0f --- /dev/null +++ b/docs/design/iter-slm305-edit-language-20260724.md @@ -0,0 +1,102 @@ +# SLM-299 (LAR1-03): X22 edit-space reachability audit + +- generated_at: `2026-07-24T20:22:57Z` +- seed: `root = Stack([], "column")` +- mode: `extended` +- max_edits: 8, node_budget: 120 +- verdict policy: reachable_fraction is computed over decided cases only; UNKNOWN_BUDGET cases are reported separately and are never counted as unreachable; suites without a corpus are corpus_unavailable, never zero-reachable. Reachability is a space-coverage proof, never a model-quality claim. + +> Reachability is space coverage, not model quality: no quality claim +> follows from these proofs alone. + +## Suite summary + +| suite | n | decided | unknown | reachable_fraction | min/med/max edits | +| --- | --- | --- | --- | --- | --- | +| train | 6 | 6 | 0 | 0.0 | — | +| smoke | 3 | 3 | 0 | 0.0 | — | +| held_out | 5 | 5 | 0 | 0.0 | — | +| adversarial | 4 | 4 | 0 | 0.0 | — | +| ood | 4 | 4 | 0 | 0.0 | — | +| rico | 6 | 6 | 0 | 0.0 | — | + +## Reason-code histograms + +- **train**: `{"needs_direction_change": 3, "unsupported_component": 3}` +- **smoke**: `{"unsupported_component": 3}` +- **held_out**: `{"unsupported_component": 5}` +- **adversarial**: `{"needs_direction_change": 3, "unsupported_component": 1}` +- **ood**: `{"unsupported_component": 4}` +- **rico**: `{"needs_direction_change": 6}` + +## Action / component coverage (reachable paths) + +- **train**: actions `{}` components `{}` +- **smoke**: actions `{}` components `{}` +- **held_out**: actions `{}` components `{}` +- **adversarial**: actions `{}` components `{}` +- **ood**: actions `{}` components `{}` +- **rico**: actions `{}` components `{}` + +## Per-case verdicts + +- **train**: + - `train_auth_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `train_auth_01_aug_dir` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `train_auth_01_syn_2` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `train_button_row_01` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `train_button_row_01_aug_dir` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `train_button_row_01_syn_0` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) +- **smoke**: + - `smoke_hero_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `smoke_button_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `smoke_callout_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) +- **held_out**: + - `held_out_form_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_dual_card_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_input_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_tabs_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `held_out_settings_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) +- **adversarial**: + - `adv_empty_prompt_01` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `adv_dual_card_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `adv_deep_nest_01` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `adv_many_buttons_01` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) +- **ood**: + - `ood_dashboard_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `ood_gallery_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `ood_modal_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) + - `ood_auth_01` PROVEN_UNREACHABLE (unsupported_component, lower_bound=None) +- **rico**: + - `rico_eval_test_0` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_1` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_2` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_4` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_8` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + - `rico_eval_test_9` PROVEN_UNREACHABLE (needs_direction_change, lower_bound=None) + +## X22 evidence annotations (append-only) + +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [adversarial]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided adversarial cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [adversarial]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided adversarial cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [held_out]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [held_out]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [ood]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [ood]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [rico]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided rico cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [rico]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided rico cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [smoke]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [smoke]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-x22-d3-kapur-tree-edit-20260717.md` [train]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided train cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on train in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. +- `docs/design/iter-efs0-04-x22-reproduction-20260717.md` [train]: SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 6 decided train cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on train in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality. + +## Old (v1) vs extended (SLM-305) reachability + +| suite | reachable v1 | reachable extended | verdict flips | action-cost delta min/med/max | +| --- | --- | --- | --- | --- | +| train | 0.0 | 0.0 | 0 | — | +| smoke | 0.0 | 0.0 | 0 | — | +| held_out | 0.0 | 0.0 | 0 | — | +| adversarial | 0.0 | 0.0 | 0 | — | +| ood | 0.0 | 0.0 | 0 | — | +| rico | 0.0 | 0.0 | 0 | — | diff --git a/docs/design/langsmith-telemetry-smoke-20260724.json b/docs/design/langsmith-telemetry-smoke-20260724.json new file mode 100644 index 000000000..9a0a5ecb3 --- /dev/null +++ b/docs/design/langsmith-telemetry-smoke-20260724.json @@ -0,0 +1,51 @@ +{ + "schema": "langsmith_telemetry_wiring/v1", + "run": { + "kind": "langsmith_telemetry_config_smoke", + "date": "2026-07-24", + "device": "local", + "steps": 0, + "context_backend": "none", + "matrix_set": "telemetry-config", + "suite_n": 0, + "honesty": "tooling_wiring_not_model_eval", + "checkpoint": null, + "ship_gates": { + "status": "not_run", + "pass": null, + "reason": "No model checkpoint or quality suite was evaluated." + } + }, + "configuration": { + "langsmith": { + "enabled": true, + "project": "slm-training", + "api_key": "configured_not_recorded", + "flush_seconds": 2.0, + "payload": "summary_only" + }, + "otlp": { + "service_name": "slm-training", + "resource_namespace": "openui", + "export_timeout_ms": 1000, + "endpoint": "not_configured" + } + }, + "verification": { + "trace_id": "fdf78e5360a6647b09fa0882a2c5d4ec", + "root_run_name": "slm.telemetry.config", + "remote_readback": true, + "remote_run_id_matches_w3c_trace_id": true + }, + "verdict": "configured_and_readback_verified", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "cc2521119d6d89d4663131a7001ce45897ca8cdd", + "code_dirty": true, + "components": { + "harness.model_build.eval": "v52", + "evals.agentv": "v3" + }, + "stamped_at": "2026-07-24T19:21:28.616031+00:00" + } +} diff --git a/package-lock.json b/package-lock.json index c31389ed7..c16585b33 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,10 @@ "": { "name": "slm-training-web", "version": "0.1.0", + "dependencies": { + "langsmith": "^0.8.7", + "openai": "^6.49.0" + }, "devDependencies": { "@agentv/core": "4.42.4", "@playwright/mcp": "0.0.78", @@ -274,7 +278,7 @@ "version": "3.975.2", "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.2.tgz", "integrity": "sha512-iyeXwziyjJpixq5OmhsIyrSWx8vwcI7gDo4yRUC3EP7NQtOo9iAJiIEc3G+/HkhtNXqOhofiCK7Lc34Sq+fJWg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.974.1", @@ -294,7 +298,7 @@ "version": "3.972.58", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.58.tgz", "integrity": "sha512-vyGtvK1rY940eq7JT0yIGKuZ+2kpPSJcHibSvGlit5oiMFDamzC7cxBGLl4FLnd6suihMXDI2FSF2dL6TmBqPA==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -311,7 +315,7 @@ "version": "3.972.60", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.60.tgz", "integrity": "sha512-g9b9YzDrD5pcKiPBJfCSXRfFMrA39eR0guUhZ5SRm+7vMAVc43+effxbcamxBjSd5bUhrdKo5te/yQuWurLXLA==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -330,7 +334,7 @@ "version": "3.973.2", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.2.tgz", "integrity": "sha512-Yr7yxNyQ8aHt9Ww0RPFUZx+xiem+vl7vuwhP0tniTijoesJNV5jou9HCgVpI0GEPAF+89TkOvilE5uRrZJnjaw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -355,7 +359,7 @@ "version": "3.972.64", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.64.tgz", "integrity": "sha512-YQoSI4d6kXvoenoG/0Jv/PqaAuukHzGmGXGyHBQYeEUNsYovlNAn/Sw1wp/WQbhcQ3HsEMGgjEahvD3igz6ecQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -373,7 +377,7 @@ "version": "3.972.68", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.68.tgz", "integrity": "sha512-4akjzW9CjorByYfqXBXmYUh/h7Io3U4DtVgGGh9TQraZ7ZlyJqNyHwDRGiUFnHD+BTOeTbCesCa4sJaK7BGZ7A==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.58", @@ -396,7 +400,7 @@ "version": "3.972.58", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.58.tgz", "integrity": "sha512-1nYitRCaDmXWUrpBJt6WlcGjLx1JVsMY8rlYuHHsTYTSaYikbixYdQSyINN2VYq1F798uTO9qHAzytL25M8g3A==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -413,7 +417,7 @@ "version": "3.973.2", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.2.tgz", "integrity": "sha512-pjMLaLU/JZi5lVfmR14V1OZqRBTuMHf6AwGNZA0K9hK+JKtO3jcLBarfD8iq5oc8cSowvc/9R32sqMVXZPo6xQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -432,7 +436,7 @@ "version": "3.972.64", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.64.tgz", "integrity": "sha512-7Buc7p0OvDHW7iBsu4b+YdS0WnaFBDGKDfbVQqaac9dkWiSiUtIoarBDsA1RmOVXZijaZJDoHJFIQiicQvWRlQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -501,7 +505,7 @@ "version": "3.997.32", "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.32.tgz", "integrity": "sha512-6Yj2fr9XF67cndITea48rchTdVr3VGx6PN47bIKNinJAjLkmaIlz/4EBPCgJ8UmhVopiXmeAuPLI3+DXDDbMhQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -521,7 +525,7 @@ "version": "3.996.40", "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.40.tgz", "integrity": "sha512-wrGZ/authosokclY1DXsiWT/1WjfCI22FuZGgdcilF+XLTXs5dCjAtiFYSPsEToZkbm3Lj2YP8PoWg0yoMNu0g==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.974.1", @@ -537,7 +541,7 @@ "version": "3.1087.0", "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1087.0.tgz", "integrity": "sha512-umM+qNq16f2fH+VLM5MqXW4ORNQAjk+TOSto73xbUHcKaU41L48j786r3UWQYlejeJk37NlvRYgxBT+MBkfaYQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "^3.975.2", @@ -555,7 +559,7 @@ "version": "3.974.1", "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.1.tgz", "integrity": "sha512-W0IQZR0eaBqlBFIIofMapaWkw1W0U+Xi4dvW+BqwmCEMd8Ng2U6IhkxuPSjMVnR8klLjfuS9PeZWUl1N6UaZdg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.16.1", @@ -569,7 +573,7 @@ "version": "3.972.35", "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.35.tgz", "integrity": "sha512-pXzaWe3evZhjxDXAlMnqISe/XefTCGwBJG4nFTXaWSgAnMkqPEhxEPqJNhhpGesEvKFhvNpnozJJ4GTL11bRYw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.16.1", @@ -583,7 +587,7 @@ "version": "0.3.0", "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=18.0.0" @@ -644,6 +648,28 @@ } } }, + "node_modules/@earendil-works/pi-ai/node_modules/openai": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.26.0.tgz", + "integrity": "sha512-zd23dbWTjiJ6sSAX6s0HrCZi41JwTA1bQVs0wLQPZ2/5o2gxOJA5wh7yOAUgwYybfhDXyhwlpeQf7Mlgx8EOCA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/@github/copilot": { "version": "1.0.70", "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.70.tgz", @@ -1458,7 +1484,6 @@ "version": "1.9.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", - "dev": true, "license": "Apache-2.0", "optional": true, "engines": { @@ -1821,7 +1846,6 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.5.1.tgz", "integrity": "sha512-iZH3Gw8cxQn0gjpOjJMmKLd9GIaNh/E3v3ST67vyzLSxHBs14HsG4dy7jMYyC5WXGdBVEcM7U/XTF5hCQxjDMw==", - "dev": true, "license": "Apache-2.0", "optional": true, "dependencies": { @@ -1840,7 +1864,6 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", - "dev": true, "license": "Apache-2.0", "optional": true, "dependencies": { @@ -1857,7 +1880,6 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.5.1.tgz", "integrity": "sha512-BViBCdE/GuXRlp9k7nS1w6wJvY5fnFX5XvuEtWsTAOQFIO89Eru7lGW3WbfbxtCuZ/GbrJfAziXG0w0dpxL7eQ==", - "dev": true, "license": "Apache-2.0", "optional": true, "dependencies": { @@ -1914,7 +1936,7 @@ "version": "1.43.0", "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=14" @@ -2055,7 +2077,7 @@ "version": "3.29.4", "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.4.tgz", "integrity": "sha512-G1GRglAabzEhqghJMBAd54FkRS7SAFGHEwbhcI9r+O+LIMuFsLyXkLZkCoFSgAglRu8s/URVXJB0hglq3ZipIg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.16.1", @@ -2069,7 +2091,7 @@ "version": "4.4.9", "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.9.tgz", "integrity": "sha512-2nfV4qRKiYeXU4zD2vvSCfg5dfp/BuhrM73vt7q9gzBhxs4rbPxXY21wo+kyI3bRmXcEGRnCLTaW8O437jzHIg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/core": "^3.29.4", @@ -2084,7 +2106,7 @@ "version": "5.6.6", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.6.tgz", "integrity": "sha512-NHLgAlORUFZjn5ZfhYuyyKMlXA1WLYOdGxEhyNxrPpbJzoacGbl0chn1lN2KiZ8mpNVk0tV5607CSYlYs/OFgw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/core": "^3.29.4", @@ -2099,7 +2121,7 @@ "version": "4.9.6", "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.6.tgz", "integrity": "sha512-odd+HYx3OLcXRSEz0ZeF3JQdSYdK8QnRgA2N87cPW7coWIbKfRk7a9VQjfeWQLqnzrDLk23KMEn46p8N7M/JFg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/core": "^3.29.4", @@ -2114,7 +2136,7 @@ "version": "5.6.5", "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.5.tgz", "integrity": "sha512-MO5VEhwVl0BN7xVoVeNrZfiUFoQtqxUbgl6/RwOTlMMxCSjblG8twSrVTwz3J4w9WZxd2rBfBAUXjH77agspBg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@smithy/core": "^3.29.4", @@ -2129,7 +2151,7 @@ "version": "4.16.1", "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -2334,7 +2356,7 @@ "version": "2.14.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/braces": { @@ -2678,6 +2700,12 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -3349,6 +3377,39 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/langsmith": { + "version": "0.8.7", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.8.7.tgz", + "integrity": "sha512-QWcc7JwmGy+sPRJwqCf9xLiiSIhwc/i7oVwjQezkx6UzyFlokME3Wxy80qKqirTyJvWG1S9ylqtxBDpFRpas5g==", + "license": "MIT", + "dependencies": { + "p-queue": "6.6.2" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } + } + }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -3554,19 +3615,27 @@ } }, "node_modules/openai": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-6.26.0.tgz", - "integrity": "sha512-zd23dbWTjiJ6sSAX6s0HrCZi41JwTA1bQVs0wLQPZ2/5o2gxOJA5wh7yOAUgwYybfhDXyhwlpeQf7Mlgx8EOCA==", - "dev": true, + "version": "6.49.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.49.0.tgz", + "integrity": "sha512-aYCc0C6L864eR6WSYIwQGyXriw/nIyZx0ObvhzOEVuk0zoBDpynjSbrionWI7q65B5H8jJX0DXR9snEzM6bfPg==", "license": "Apache-2.0", - "bin": { - "openai": "bin/cli" - }, "peerDependencies": { + "@aws-sdk/credential-provider-node": ">=3.972.0 <4", + "@smithy/hash-node": ">=4.3.0 <5", + "@smithy/signature-v4": ">=5.4.0 <6", "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "peerDependenciesMeta": { + "@aws-sdk/credential-provider-node": { + "optional": true + }, + "@smithy/hash-node": { + "optional": true + }, + "@smithy/signature-v4": { + "optional": true + }, "ws": { "optional": true }, @@ -3575,6 +3644,31 @@ } } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-retry": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", @@ -3589,6 +3683,18 @@ "node": ">=8" } }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -4127,7 +4233,7 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, + "devOptional": true, "license": "0BSD" }, "node_modules/type-is": { @@ -4244,7 +4350,7 @@ "version": "8.21.1", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=10.0.0" @@ -4282,7 +4388,7 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", - "dev": true, + "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/package.json b/package.json index 05e70d356..1d4c0d837 100644 --- a/package.json +++ b/package.json @@ -27,5 +27,9 @@ }, "engines": { "node": ">=20 <23" + }, + "dependencies": { + "langsmith": "^0.8.7", + "openai": "^6.49.0" } } diff --git a/pyproject.toml b/pyproject.toml index 36eb95517..3fd1067ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,10 @@ research = [ "openai>=2.0,<3", "trackio>=0.1,<1", ] +observability = [ + "langsmith>=0.9,<1", + "python-dotenv>=1.0,<2", +] web = [ "fastapi>=0.110,<1", "onnx>=1.16,<2", diff --git a/scripts/run_agentv_eval.mjs b/scripts/run_agentv_eval.mjs index 7d24a7ba1..5a0d73657 100644 --- a/scripts/run_agentv_eval.mjs +++ b/scripts/run_agentv_eval.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node import { pathToFileURL } from "node:url"; +import { randomUUID } from "node:crypto"; function option(name) { const index = process.argv.indexOf(name); @@ -10,10 +11,62 @@ function option(name) { return process.argv[index + 1]; } +function optional(name) { + const index = process.argv.indexOf(name); + return index === -1 ? undefined : process.argv[index + 1]; +} + +function langsmithEnabled() { + return /^(1|true|yes|on)$/i.test(process.env.LANGSMITH_TRACING ?? "") + && Boolean(process.env.LANGSMITH_API_KEY); +} + +function traceUuid(traceId) { + return traceId && /^[0-9a-f]{32}$/i.test(traceId) + ? `${traceId.slice(0, 8)}-${traceId.slice(8, 12)}-${traceId.slice(12, 16)}-${traceId.slice(16, 20)}-${traceId.slice(20)}` + : undefined; +} + +async function publishLangSmithSummary({ traceId, runId, experiment, result }) { + const parentRunId = traceUuid(traceId); + if (!langsmithEnabled() || !parentRunId) return; + try { + const { Client } = await import("langsmith"); + const client = new Client({ + apiKey: process.env.LANGSMITH_API_KEY, + apiUrl: process.env.LANGSMITH_ENDPOINT, + workspaceId: process.env.LANGSMITH_WORKSPACE_ID, + omitTracedRuntimeInfo: true, + }); + const now = new Date().toISOString(); + await client.createRun({ + id: randomUUID(), + trace_id: parentRunId, + parent_run_id: parentRunId, + project_name: process.env.LANGSMITH_PROJECT || "slm-training", + name: "agentv.publication", + run_type: "tool", + inputs: { run_id: runId, experiment }, + outputs: { summary: result.summary }, + start_time: now, + end_time: now, + extra: { metadata: { w3c_trace_id: traceId, sdk: "@agentv/core" } }, + }); + await Promise.race([ + client.flush(), + new Promise((resolve) => setTimeout(resolve, 500)), + ]); + } catch (error) { + console.warn(`LangSmith AgentV export failed: ${String(error)}`); + } +} + const specFile = option("--spec"); const outputDir = option("--output-dir"); const experiment = option("--experiment"); const sdkRoot = option("--sdk-root"); +const traceId = optional("--trace-id"); +const runId = optional("--run-id"); const sdkUrl = pathToFileURL( `${sdkRoot}/node_modules/@agentv/core/dist/index.js`, ); @@ -29,6 +82,8 @@ const result = await evaluate({ experiment, }); +await publishLangSmithSummary({ traceId, runId, experiment, result }); + console.log(JSON.stringify({ summary: result.summary, artifacts: result.artifacts, diff --git a/scripts/run_slm299_reachability_audit.py b/scripts/run_slm299_reachability_audit.py index 73dcbf04c..5b65e5c04 100644 --- a/scripts/run_slm299_reachability_audit.py +++ b/scripts/run_slm299_reachability_audit.py @@ -87,7 +87,7 @@ def load_corpora() -> dict[str, list[dict[str, Any]]]: def analyze_record( - record: dict[str, Any], *, max_edits: int, node_budget: int + record: dict[str, Any], *, max_edits: int, node_budget: int, mode: str = "extended" ) -> ReachabilityCase: target = str(record.get("openui") or "") placeholders = record.get("placeholders") or extract_placeholders(target) @@ -97,6 +97,7 @@ def analyze_record( slot_inventory=[str(p) for p in placeholders], max_edits=max_edits, node_budget=node_budget, + mode=mode, ) @@ -185,6 +186,54 @@ def build_x22_evidence_annotations( return annotations +def _summarize_records( + records: list[dict[str, Any]], *, max_edits: int, node_budget: int, mode: str +) -> dict[str, Any]: + cases = [ + ( + str(record.get("id", f"case_{index}")), + analyze_record(record, max_edits=max_edits, node_budget=node_budget, mode=mode), + ) + for index, record in enumerate(records) + ] + return {"status": "ok", **summarize_suite(cases)} + + +def _compare_summaries(v1: dict[str, Any], extended: dict[str, Any]) -> dict[str, Any]: + """Old-vs-extended comparison: verdict flips and action-cost deltas over + matched cases (never a quality claim — reachability only).""" + v1_cases = {c["id"]: c for c in v1.get("cases", [])} + ext_cases = {c["id"]: c for c in extended.get("cases", [])} + deltas: list[int] = [] + flips: list[dict[str, Any]] = [] + for case_id, ext in sorted(ext_cases.items()): + old = v1_cases.get(case_id) + if old is None: + continue + if old["verdict"] != ext["verdict"]: + flips.append( + {"id": case_id, "v1": old["verdict"], "extended": ext["verdict"]} + ) + if old["edit_lower_bound"] is not None and ext["edit_lower_bound"] is not None: + deltas.append(ext["edit_lower_bound"] - old["edit_lower_bound"]) + return { + "n_verdict_flips": len(flips), + "verdict_flips": flips, + "reachable_fraction_v1": v1.get("reachable_fraction"), + "reachable_fraction_extended": extended.get("reachable_fraction"), + "action_cost_delta": ( + { + "min": min(deltas), + "median": statistics.median(deltas), + "max": max(deltas), + "mean": round(statistics.mean(deltas), 4), + } + if deltas + else None + ), + } + + def build_report( corpora: dict[str, list[dict[str, Any]]], *, @@ -192,8 +241,12 @@ def build_report( node_budget: int, generated_at: str, limit: int | None = None, + mode: str = "extended", + compare: bool = False, ) -> dict[str, Any]: suites: dict[str, Any] = {} + suites_v1: dict[str, Any] = {} + comparisons: dict[str, Any] = {} for suite, records in corpora.items(): if not records: suites[suite] = { @@ -201,22 +254,25 @@ def build_report( "n_cases": 0, "reachable_fraction": None, } + if compare: + suites_v1[suite] = dict(suites[suite]) continue if limit is not None: records = records[:limit] - cases = [ - ( - str(record.get("id", f"{suite}_{index}")), - analyze_record(record, max_edits=max_edits, node_budget=node_budget), + suites[suite] = _summarize_records( + records, max_edits=max_edits, node_budget=node_budget, mode=mode + ) + if compare: + suites_v1[suite] = _summarize_records( + records, max_edits=max_edits, node_budget=node_budget, mode="v1" ) - for index, record in enumerate(records) - ] - suites[suite] = {"status": "ok", **summarize_suite(cases)} + comparisons[suite] = _compare_summaries(suites_v1[suite], suites[suite]) payload: dict[str, Any] = { - "schema": "slm299_edit_reachability_audit/v1", + "schema": "slm299_edit_reachability_audit/v2", "experiment_id": EXPERIMENT_ID, "seed_source": DEFAULT_SEED_SOURCE, + "mode": mode, "max_edits": max_edits, "node_budget": node_budget, "generated_at": generated_at, @@ -224,11 +280,15 @@ def build_report( "reachable_fraction is computed over decided cases only; " "UNKNOWN_BUDGET cases are reported separately and are never counted " "as unreachable; suites without a corpus are corpus_unavailable, " - "never zero-reachable." + "never zero-reachable. Reachability is a space-coverage proof, " + "never a model-quality claim." ), "suites": suites, "version_stamp": build_version_stamp(COMPONENT), } + if compare: + payload["suites_v1"] = suites_v1 + payload["old_vs_extended"] = comparisons payload["x22_evidence_annotations"] = build_x22_evidence_annotations( suites, generated_at=generated_at ) @@ -241,9 +301,13 @@ def render_markdown(payload: dict[str, Any]) -> str: "", f"- generated_at: `{payload['generated_at']}`", f"- seed: `{payload['seed_source']}`", + f"- mode: `{payload.get('mode', 'extended')}`", f"- max_edits: {payload['max_edits']}, node_budget: {payload['node_budget']}", f"- verdict policy: {payload['verdict_policy']}", "", + "> Reachability is space coverage, not model quality: no quality claim", + "> follows from these proofs alone.", + "", "## Suite summary", "", "| suite | n | decided | unknown | reachable_fraction | min/med/max edits |", @@ -291,6 +355,30 @@ def render_markdown(payload: dict[str, Any]) -> str: lines += ["", "## X22 evidence annotations (append-only)", ""] for note in payload["x22_evidence_annotations"]: lines.append(f"- `{note['target_doc']}` [{note['suite']}]: {note['annotation']}") + if "old_vs_extended" in payload: + lines += ["", "## Old (v1) vs extended (SLM-305) reachability", ""] + lines.append( + "| suite | reachable v1 | reachable extended | verdict flips | " + "action-cost delta min/med/max |" + ) + lines.append("| --- | --- | --- | --- | --- |") + for suite, comp in payload["old_vs_extended"].items(): + delta = comp["action_cost_delta"] or {} + delta_txt = ( + f"{delta.get('min')}/{delta.get('median')}/{delta.get('max')}" + if delta + else "—" + ) + lines.append( + f"| {suite} | {comp['reachable_fraction_v1']} | " + f"{comp['reachable_fraction_extended']} | " + f"{comp['n_verdict_flips']} | {delta_txt} |" + ) + for suite, comp in payload["old_vs_extended"].items(): + for flip in comp["verdict_flips"]: + lines.append( + f"- flip `{suite}/{flip['id']}`: {flip['v1']} → {flip['extended']}" + ) lines.append("") return "\n".join(lines) @@ -300,6 +388,17 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--max-edits", type=int, default=8) parser.add_argument("--node-budget", type=int, default=800) parser.add_argument("--limit", type=int, default=None, help="per-suite case cap (debug)") + parser.add_argument( + "--mode", + choices=["v1", "extended"], + default="extended", + help="action space to audit (extended is the deployed SLM-305 space)", + ) + parser.add_argument( + "--compare", + action="store_true", + help="also run the v1 space and publish old-vs-extended deltas", + ) parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) args = parser.parse_args(argv) @@ -311,6 +410,8 @@ def main(argv: list[str] | None = None) -> int: node_budget=args.node_budget, generated_at=generated_at, limit=args.limit, + mode=args.mode, + compare=args.compare, ) args.json_out.parent.mkdir(parents=True, exist_ok=True) args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") diff --git a/src/slm_training/evals/agentv.py b/src/slm_training/evals/agentv.py index 56ba94f24..ca59544fc 100644 --- a/src/slm_training/evals/agentv.py +++ b/src/slm_training/evals/agentv.py @@ -101,8 +101,7 @@ def publish_agentv_evaluation( spec_path.write_text("\n".join(rows) + "\n", encoding="utf-8") runner, runtime_root = _agentv_runtime(repo_root) - completed = subprocess.run( - [ + command = [ "node", str(runner), "--spec", @@ -113,7 +112,12 @@ def publish_agentv_evaluation( slug, "--sdk-root", str(runtime_root), - ], + ] + trace_id = _run_trace_id(Path(run_dir)) + if trace_id is not None: + command.extend(("--trace-id", trace_id, "--run-id", Path(run_dir).name)) + completed = subprocess.run( + command, cwd=runtime_root, check=False, capture_output=True, @@ -182,6 +186,18 @@ def _read_agentv_criterion_results(published: dict[str, Any]) -> list[dict[str, return results +def _run_trace_id(run_dir: Path) -> str | None: + """Return the valid W3C trace ID associated with a local run, if present.""" + path = run_dir / "trace.json" + if not path.is_file(): + return None + try: + trace_id = str(json.loads(path.read_text(encoding="utf-8")).get("trace_id", "")) + except (OSError, json.JSONDecodeError): + return None + return trace_id if re.fullmatch(r"[0-9a-f]{32}", trace_id) else None + + def _stamp_agentv_artifacts( output_dir: Path, version_stamp: dict[str, Any] ) -> None: diff --git a/src/slm_training/harnesses/experiments/slm299_edit_reachability.py b/src/slm_training/harnesses/experiments/slm299_edit_reachability.py index 537776b07..7d666819e 100644 --- a/src/slm_training/harnesses/experiments/slm299_edit_reachability.py +++ b/src/slm_training/harnesses/experiments/slm299_edit_reachability.py @@ -33,16 +33,26 @@ from slm_training.dsl.parser import validate from slm_training.models.tree_edit_diffusion import ( ACTION_ADD, + ACTION_ADD_CONTAINER, + ACTION_BIND_PLACEHOLDER, + ACTION_INSERT_STATEMENT, + ACTION_INSERT_SUBTREE, ACTION_REMOVE, + ACTION_REMOVE_CONTAINER, ACTION_REPLACE, + ACTION_REPLACE_STATEMENT, + ACTION_REPLACE_SUBTREE, CONTAINER_COMPONENTS, + CONTAINER_RESTS, LEAF_COMPONENTS, MAX_SLOTS, + V05_TEMPLATES, Edit, Statement, TreeEditSpace, parse_statements, render_statements, + v05_template_index, ) __all__ = [ @@ -111,9 +121,9 @@ def add_container_action( components: Sequence[str] = CONTAINER_COMPONENTS, ) -> ExtraAction: """Synthetic ``ADD_CONTAINER``: append a fresh empty container statement - and reference it from an existing container. Does **not** exist in the - real X22 space (ADD creates leaves only); used to prove that a target is - unreachable *because* the space cannot create containers. + and reference it from an existing container. Retired what-if lane: the + extended (SLM-305) space has the real ``ACTION_ADD_CONTAINER``, so this + synthetic action is only meaningful in ``mode="v1"`` analyses. """ def _generate( @@ -201,13 +211,22 @@ def _normalize_inventory(slot_inventory: Sequence[str]) -> list[str]: return inventory[:MAX_SLOTS] -def _is_unsupported_pack_feature(target_source: str) -> bool: +def _is_unsupported_pack_feature( + target_source: str, *, extended: bool = False +) -> bool: if any(marker in target_source for marker in _V05_TEXT_MARKERS): return True for line in target_source.splitlines(): line = line.strip() if line and _V05_LINE_RE.match(line): - return True + if not extended: + return True + # SLM-305: the extended space models V0.5 statements through the + # canonical-template statement actions; a V0.5 line that does not + # instantiate a canonical template is still unsupported. + stmt = parse_statements(line) + if stmt is None or len(stmt) != 1 or v05_template_index(stmt[0]) is None: + return True return False @@ -260,24 +279,41 @@ def _check_invariants( inventory: list[str], space: TreeEditSpace, capabilities: frozenset[str] = frozenset(), + *, + extended: bool = False, ) -> str | None: - """Structural impossibility proofs over the EXACT X22 action set. - - REPLACE preserves container-ness, arity, and the container's enum/direction - arg; ADD creates leaves only and binds only inventory slots; REMOVE deletes - leaves only. Anything the target needs beyond that is proven unreachable. + """Structural impossibility proofs over the EXACT action set. + + v1 space: REPLACE preserves container-ness, arity, and the container's + enum/direction arg; ADD creates leaves only and binds only inventory + slots; REMOVE deletes leaves only. Extended (SLM-305) space: ADD_CONTAINER + / INSERT_SUBTREE mint containers carrying one of ``CONTAINER_RESTS`` and + bind only inventory slots, REPLACE_SUBTREE / BIND_PLACEHOLDER rebind + leaves only to + inventory slots, and the V0.5 statement actions mint only canonical + templates — the invariant reasons fire only when the corresponding REAL + action of the analyzed mode is absent. """ seed_containers = [s for s in seed if s.has_list] target_containers = [s for s in target if s.has_list] - target_leaves = [s for s in target if not s.has_list] - - # No action creates a container (ADD is leaf-only). + # V0.5 canonical-template statements are produced by the statement actions + # in the extended space; they are not UI leaves/containers. + target_leaves = [ + s + for s in target + if not s.has_list and not (extended and v05_template_index(s) is not None) + ] + + # No action creates a container in the v1 space (ADD is leaf-only); the + # extended space has the real ADD_CONTAINER / INSERT_SUBTREE. if ( len(target_containers) > len(seed_containers) and "container_add" not in capabilities ): return REASON_NEEDS_CONTAINER_ADD - # No action removes a container (REMOVE is leaf-only). + # No action removes a container (REMOVE is leaf-only); REMOVE_CONTAINER + # can only remove non-root containers, and both seed and target keep root, + # so this invariant holds in every mode. if len(target_containers) < len(seed_containers): return REASON_NEEDS_CONTAINER_REMOVE @@ -289,9 +325,9 @@ def _check_invariants( if stmt.comp not in LEAF_COMPONENTS or stmt.comp not in known: return REASON_UNSUPPORTED_COMPONENT - # ADD binds only inventory slots; REPLACE cannot change a leaf's bound - # slot, and the seed carries no leaves. So every target leaf slot must - # come from the prompt inventory. + # ADD / INSERT_SUBTREE / REPLACE_SUBTREE / BIND_PLACEHOLDER bind only + # inventory slots, and the seed carries no leaves. So every target leaf + # slot must come from the prompt inventory in every mode. for stmt in target_leaves: slot = _leaf_slot(stmt.rest) if slot is None: @@ -301,14 +337,24 @@ def _check_invariants( return REASON_NEEDS_SLOT_REBIND # REPLACE preserves the container's raw enum/direction arg text (rest); - # no real action edits it. Every target container must therefore carry - # the seed container's rest. With a synthetic container-creating action - # (which mints containers carrying the seed's rest), the same per-container - # rule applies; without it the multisets must match exactly. + # no real action edits it. Containers minted by the container-creating + # actions carry exactly one of CONTAINER_RESTS. Every target container + # rest must therefore be a seed rest or (with container_add) a candidate + # mint rest; without any container-creating action the multisets must + # match exactly. seed_rests = sorted(s.rest for s in seed_containers) target_rests = sorted(s.rest for s in target_containers) + # Root can never be removed or re-minted, and REPLACE preserves rest: + # the target root's rest must equal the seed root's rest in every mode. + seed_root_rest = next( + (s.rest for s in seed_containers if s.name == "root"), None + ) + for stmt in target_containers: + if stmt.name == "root" and stmt.rest != seed_root_rest: + return REASON_NEEDS_DIRECTION_CHANGE if "container_add" in capabilities: - if any(rest not in seed_rests for rest in target_rests): + allowed = set(seed_rests) | set(CONTAINER_RESTS) + if any(rest not in allowed for rest in target_rests): return REASON_NEEDS_DIRECTION_CHANGE elif seed_rests != target_rests: return REASON_NEEDS_DIRECTION_CHANGE @@ -320,17 +366,45 @@ def _enumerate_children( space: TreeEditSpace, statements: list[Statement], inventory: list[str], + *, + mode: str = "extended", + visited: set[str] | None = None, ) -> list[tuple[list[Statement], dict[str, Any]]]: - """All one-edit successors under the REAL X22 action set, applied through - ``TreeEditSpace.apply`` so preconditions and parser re-validation are the - deployed ones by construction.""" + """All one-edit successors under the REAL action set of ``mode``, applied + through ``TreeEditSpace.apply`` so preconditions and parser re-validation + are the deployed ones by construction. ``v1`` enumerates the original + REPLACE/ADD/REMOVE set; ``extended`` (SLM-305) adds the container, + subtree, V0.5-statement, and placeholder-binding actions. Deterministic: + same state and mode always yield the same enumeration order. + + ``visited`` (canonical-key set) is a pure search-efficiency hook: states + already keyed are rejected pre-validation inside ``apply``; it never + changes WHICH distinct states are reachable.""" children: list[tuple[list[Statement], dict[str, Any]]] = [] n_comp = len(space.components) n_slots = min(len(inventory), MAX_SLOTS) + leaf_comp_idxs = [ + i for i, c in enumerate(space.components) if c in LEAF_COMPONENTS + ] + container_comp_idxs = [ + i for i, c in enumerate(space.components) if c in CONTAINER_COMPONENTS + ] + pre = None + if visited is not None: + pre = lambda working: _canonical_key(working) not in visited # noqa: E731 + by_name = {s.name: s for s in statements} + + def _replace_subtree_ok(stmt: Statement) -> bool: + if not stmt.has_list or len(stmt.children) != 1: + return False + leaf = by_name.get(stmt.children[0]) + return leaf is not None and not leaf.has_list + for stmt_idx in range(len(statements)): + stmt = statements[stmt_idx] for comp_idx in range(n_comp): edit = Edit(ACTION_REPLACE, stmt_idx, comp_idx) - nxt = space.apply(statements, edit, inventory) + nxt = space.apply(statements, edit, inventory, pre) if nxt is not None: children.append( ( @@ -344,7 +418,7 @@ def _enumerate_children( ) for slot_idx in range(n_slots): edit = Edit(ACTION_ADD, stmt_idx, comp_idx, slot_idx) - nxt = space.apply(statements, edit, inventory) + nxt = space.apply(statements, edit, inventory, pre) if nxt is not None: children.append( ( @@ -357,10 +431,124 @@ def _enumerate_children( }, ) ) + if mode == "v1": + edit = Edit(ACTION_REMOVE, stmt_idx) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append((nxt, {"action": "REMOVE", "stmt": stmt_idx})) + continue + # SLM-305 extended real actions. + if stmt.has_list: + for comp_idx in container_comp_idxs: + for rest_idx in range(len(CONTAINER_RESTS)): + edit = Edit(ACTION_ADD_CONTAINER, stmt_idx, comp_idx, + target=rest_idx) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "ADD_CONTAINER", + "stmt": stmt_idx, + "comp": space.components[comp_idx], + "rest": CONTAINER_RESTS[rest_idx], + }, + ) + ) + for comp_idx in container_comp_idxs: + for slot_idx in range(n_slots): + for payload in leaf_comp_idxs: + for rest_idx in range(len(CONTAINER_RESTS)): + edit = Edit( + ACTION_INSERT_SUBTREE, stmt_idx, comp_idx, slot_idx, + target=rest_idx, payload=payload, + ) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "INSERT_SUBTREE", + "stmt": stmt_idx, + "comp": space.components[comp_idx], + "slot": inventory[slot_idx], + "leaf_comp": space.components[payload], + "rest": CONTAINER_RESTS[rest_idx], + }, + ) + ) + if _replace_subtree_ok(stmt): + for slot_idx in range(n_slots): + for payload in leaf_comp_idxs: + edit = Edit( + ACTION_REPLACE_SUBTREE, stmt_idx, slot=slot_idx, + payload=payload, + ) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "REPLACE_SUBTREE", + "stmt": stmt_idx, + "slot": inventory[slot_idx], + "leaf_comp": space.components[payload], + }, + ) + ) + else: + for slot_idx in range(n_slots): + edit = Edit(ACTION_BIND_PLACEHOLDER, stmt_idx, slot=slot_idx) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "BIND_PLACEHOLDER", + "stmt": stmt_idx, + "slot": inventory[slot_idx], + }, + ) + ) + edit = Edit(ACTION_REMOVE_CONTAINER, stmt_idx) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + (nxt, {"action": "REMOVE_CONTAINER", "stmt": stmt_idx}) + ) edit = Edit(ACTION_REMOVE, stmt_idx) - nxt = space.apply(statements, edit, inventory) + nxt = space.apply(statements, edit, inventory, pre) if nxt is not None: children.append((nxt, {"action": "REMOVE", "stmt": stmt_idx})) + for payload in range(len(V05_TEMPLATES)): + edit = Edit(ACTION_REPLACE_STATEMENT, stmt_idx, payload=payload) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "REPLACE_STATEMENT", + "stmt": stmt_idx, + "template": payload, + }, + ) + ) + if mode != "v1": + for payload in range(len(V05_TEMPLATES)): + edit = Edit(ACTION_INSERT_STATEMENT, payload=payload) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + {"action": "INSERT_STATEMENT", "template": payload}, + ) + ) return children @@ -372,14 +560,19 @@ def analyze_reachability( max_edits: int = 8, extra_actions: Sequence[ExtraAction] = (), node_budget: int = 800, + mode: str = "extended", ) -> ReachabilityCase: """Prove (or honestly fail to prove) reachability of ``target_source`` - from ``seed_source`` under the real X22 tree-edit space. + from ``seed_source`` under the real tree-edit space of ``mode``. - ``extra_actions`` are hypothetical transitions (what-if analysis only); - when any appear on a found path the case is marked in ``details`` so the - proof is never confused with the real space. + ``mode="v1"`` analyzes the original SLM-299 action set (REPLACE / ADD / + REMOVE); ``mode="extended"`` (default, SLM-305) analyzes the deployed + extended edit language. ``extra_actions`` are hypothetical transitions + (what-if analysis only); when any appear on a found path the case is + marked in ``details`` so the proof is never confused with the real space. """ + if mode not in {"v1", "extended"}: + raise ValueError(f"unknown reachability mode {mode!r}") space = _shared_space() inventory = _normalize_inventory(slot_inventory) @@ -396,9 +589,10 @@ def analyze_reachability( "node_budget": node_budget, "inventory_size": len(inventory), "extra_actions": [a.name for a in extra_actions], + "mode": mode, } - if _is_unsupported_pack_feature(target_source): + if _is_unsupported_pack_feature(target_source, extended=(mode == "extended")): return ReachabilityCase( verdict=Verdict.PROVEN_UNREACHABLE, reason_code=REASON_UNSUPPORTED_PACK_FEATURE, @@ -423,7 +617,14 @@ def analyze_reachability( capabilities = frozenset().union( *(a.capabilities for a in extra_actions) ) if extra_actions else frozenset() - fired = _check_invariants(target, seed, inventory, space, capabilities) + if mode == "extended": + # ADD_CONTAINER / INSERT_SUBTREE are REAL actions in the extended + # space, so the container_add invariant must not fire; the synthetic + # add_container_action what-if lane is retired for this space. + capabilities = capabilities | frozenset({"container_add"}) + fired = _check_invariants( + target, seed, inventory, space, capabilities, extended=(mode == "extended") + ) if fired is not None: return ReachabilityCase( verdict=Verdict.PROVEN_UNREACHABLE, @@ -456,7 +657,9 @@ def analyze_reachability( if depth >= max_edits: live_at_budget = True continue - children = _enumerate_children(space, statements, inventory) + children = _enumerate_children( + space, statements, inventory, mode=mode, visited=visited + ) for extra in extra_actions: for nxt, action in extra.generate(statements, inventory): children.append((nxt, {**action, "synthetic": True})) diff --git a/src/slm_training/harnesses/model_build/eval_runner.py b/src/slm_training/harnesses/model_build/eval_runner.py index 645808776..9ec6ba4fd 100644 --- a/src/slm_training/harnesses/model_build/eval_runner.py +++ b/src/slm_training/harnesses/model_build/eval_runner.py @@ -48,6 +48,13 @@ from slm_training.versioning import component_version _COMPONENT_RE = re.compile(r"\b([A-Z][A-Za-z0-9]*)\s*\(") +_LANGSMITH_METRIC_KEYS = ( + "n", + "parse_rate", + "placeholder_fidelity", + "structural_similarity", + "reward_score", +) def _evaluation_version_components(config: ModelBuildConfig) -> tuple[str, ...]: @@ -62,6 +69,43 @@ def _evaluation_version_components(config: ModelBuildConfig) -> tuple[str, ...]: return components + (("model.twotower",) if config.model_name == "twotower" else ()) +def _record_langsmith_evaluation(config, *, suites: dict[str, dict], scoreboard: dict) -> None: + """Publish only aggregate evaluation data to the active summary trace.""" + from slm_training.runtime.telemetry import current_trace + + trace = current_trace() + if trace is None: + return + summary = { + suite: { + key: metrics[key] + for key in _LANGSMITH_METRIC_KEYS + if key in metrics + } + for suite, metrics in suites.items() + } + trace.record_summary( + "evaluation.summary", + inputs={"run_id": config.run_id, "suites": sorted(suites)}, + outputs={ + "suites": summary, + "gates": scoreboard.get("gates"), + "agentv": scoreboard.get("agentv"), + }, + metadata={ + key: scoreboard.get(key) + for key in ( + "run_class", + "checkpoint_sha256", + "eval_data_manifest_sha", + "code_git_sha", + "version_stamp", + ) + if scoreboard.get(key) is not None + }, + ) + + def _annotate_decode_trace_records( stats: object, records: list[ExampleRecord], @@ -1706,6 +1750,19 @@ def mean(key: str) -> float: suite_path.write_text(payload, encoding="utf-8") if config.suite == "smoke": (run_dir / "eval.json").write_text(payload, encoding="utf-8") + if publish_agentv: + _record_langsmith_evaluation( + config, + suites={config.suite: metrics}, + scoreboard={ + "run_class": config.run_class, + "checkpoint_sha256": metrics.get("checkpoint_sha256"), + "eval_data_manifest_sha": metrics.get("eval_data_manifest_sha"), + "code_git_sha": metrics.get("code_git_sha"), + "version_stamp": metrics["version_stamp"], + "agentv": metrics.get("agentv"), + }, + ) return metrics @@ -1830,4 +1887,5 @@ def evaluate_suites( ) } path.write_text(json.dumps(scoreboard, indent=2) + "\n", encoding="utf-8") + _record_langsmith_evaluation(config, suites=board, scoreboard=scoreboard) return scoreboard diff --git a/src/slm_training/models/checkpoint_migrate.py b/src/slm_training/models/checkpoint_migrate.py index 5342cef2d..46e738705 100644 --- a/src/slm_training/models/checkpoint_migrate.py +++ b/src/slm_training/models/checkpoint_migrate.py @@ -293,3 +293,97 @@ def migrate_to_shared_recursive_denoiser( report_path = new_path.with_suffix(".migrate.json") report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") return report + + +def migrate_tree_edit_checkpoint( + *, + source_checkpoint: Path | str, + output_checkpoint: Path | str, + device: str = "cpu", +) -> dict: + """Warm-start a tree-edit-diffusion format-1 checkpoint to format 2. + + SLM-305 grew ``action_head`` from 4 to ``N_ACTIONS`` rows (extended edit + language). All same-shape tensors are copied verbatim; the old action-head + rows are preserved in the first rows of the new head and the new action + rows stay randomly initialized. A ``.migrate.json`` report is written next + to the output checkpoint. + """ + from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + ) + + source_checkpoint = Path(source_checkpoint) + output_checkpoint = Path(output_checkpoint) + payload = torch.load(source_checkpoint, map_location=device, weights_only=False) + if payload.get("kind") != "tree_edit_diffusion": + raise ValueError( + f"checkpoint kind {payload.get('kind')!r} is not tree_edit_diffusion" + ) + source_format = int(payload.get("format_version") or 1) + if source_format >= TreeEditDiffusionModel.CHECKPOINT_FORMAT: + raise ValueError( + f"tree-edit checkpoint is already format {source_format}; " + "migration only upgrades older formats" + ) + tokenizer_path = source_checkpoint.with_suffix(".tokenizer.json") + if not tokenizer_path.exists(): + raise FileNotFoundError( + f"missing tokenizer next to checkpoint: {tokenizer_path}" + ) + tokenizer = OpenUITokenizer.load(tokenizer_path, allow_legacy=True) + raw_config = dict(payload.get("config") or {}) + valid = set(TreeEditDiffusionConfig.__dataclass_fields__) + config = TreeEditDiffusionConfig( + **{key: value for key, value in raw_config.items() if key in valid} + ) + model = TreeEditDiffusionModel(tokenizer, config=config, device=device) + old_state = payload.get("state_dict") or {} + new_state = model.state_dict() + copied_keys: list[str] = [] + skipped_old_keys: list[str] = [] + initialized_keys: list[str] = [] + preserved_action_rows = 0 + for key, tensor in old_state.items(): + if key in new_state and new_state[key].shape == tensor.shape: + new_state[key] = tensor + copied_keys.append(key) + elif ( + key == "policy.action_head.weight" + and key in new_state + and tensor.shape[1] == new_state[key].shape[1] + and tensor.shape[0] <= new_state[key].shape[0] + ): + new_state[key][: tensor.shape[0]] = tensor + copied_keys.append(key) + preserved_action_rows = int(tensor.shape[0]) + elif ( + key == "policy.action_head.bias" + and key in new_state + and tensor.shape[0] <= new_state[key].shape[0] + ): + new_state[key][: tensor.shape[0]] = tensor + copied_keys.append(key) + else: + skipped_old_keys.append(key) + for key in new_state: + if key not in copied_keys: + initialized_keys.append(key) + model.load_state_dict(new_state, strict=True) + model.save(output_checkpoint) + report = { + "source_checkpoint": str(source_checkpoint), + "output_checkpoint": str(output_checkpoint), + "source_format_version": source_format, + "output_format_version": TreeEditDiffusionModel.CHECKPOINT_FORMAT, + "warm_start_only": True, + "preserved_action_head_rows": preserved_action_rows, + "copied_keys": copied_keys, + "skipped_old_keys": skipped_old_keys, + "initialized_keys": initialized_keys, + } + output_checkpoint.with_suffix(".migrate.json").write_text( + json.dumps(report, indent=2) + "\n", encoding="utf-8" + ) + return report diff --git a/src/slm_training/models/tree_edit_diffusion.py b/src/slm_training/models/tree_edit_diffusion.py index 26e032cf8..13ffc430d 100644 --- a/src/slm_training/models/tree_edit_diffusion.py +++ b/src/slm_training/models/tree_edit_diffusion.py @@ -23,14 +23,15 @@ import random import re from dataclasses import asdict, dataclass +from functools import lru_cache from pathlib import Path -from typing import Any +from typing import Any, Callable import torch import torch.nn as nn import torch.nn.functional as F -from slm_training.dsl.parser import validate +from slm_training.dsl.parser import validate, validate_output from slm_training.dsl.placeholders import extract_placeholders from slm_training.dsl.schema import ExampleRecord from slm_training.harnesses.model_build.plugin import GenerationRequest @@ -49,7 +50,18 @@ ACTION_REPLACE = 1 # swap the component type of one statement ACTION_ADD = 2 # add a fresh leaf statement + reference it from a container ACTION_REMOVE = 3 # remove one leaf statement and its references -N_ACTIONS = 4 +# SLM-305 (LAR2-01) extended valid-state edit language. Every new action is +# bounded, deterministic, validity-preserving (re-validated through the real +# parser before acceptance), and invertible; preconditions are documented at +# each branch of ``TreeEditSpace.apply``. +ACTION_ADD_CONTAINER = 4 # insert an empty container into a container's children +ACTION_REMOVE_CONTAINER = 5 # safe inverse: remove an empty/leaf-only container subtree +ACTION_INSERT_SUBTREE = 6 # transactional: declare container+leaf subtree and reference it +ACTION_REPLACE_SUBTREE = 7 # replace a canonical one-leaf container subtree's leaf +ACTION_INSERT_STATEMENT = 8 # insert a canonical V0.5 state/query/mutation statement +ACTION_REPLACE_STATEMENT = 9 # swap one canonical V0.5 statement for another +ACTION_BIND_PLACEHOLDER = 10 # (re)bind a leaf's slot to an inventory placeholder +N_ACTIONS = 11 MAX_STMTS = 24 MAX_SLOTS = 16 @@ -60,6 +72,40 @@ LEAF_COMPONENTS = ("TextContent", "Button", "Image", "TextInput") CONTAINER_COMPONENTS = ("Stack", "Card", "Form") +# Canonical rest texts for containers minted by ADD_CONTAINER / INSERT_SUBTREE. +# Fixed candidate set (indexed by ``Edit.target``) so minted containers are +# deterministic and the reachability invariants can reason about them exactly. +CONTAINER_RESTS: tuple[str, ...] = (', "column"', "") +CONTAINER_REST = CONTAINER_RESTS[0] + +# V0.5 statement component names (state/query/mutation/action pack forms). +V05_COMPONENTS = ("Query", "Mutation", "Action", "State", "Resource") + +# Bounded canonical V0.5 statement templates: (component, canonical arg text). +# Construction is canonical-AST-backed: each inserted/replaced line is built +# from this structured spec and fragment-validated through the canonical +# grammar (``validate_output(..., kind="statement")``) before acceptance — +# never regex string surgery on existing program text. +V05_TEMPLATES: tuple[tuple[str, str], ...] = ( + ("Query", '"tool", {arg: $x}, {default: []}, 15'), + ("Mutation", '"tool", {arg: $x}'), +) + + +def v05_template_index(stmt: Statement) -> int | None: + """Index of the canonical V0.5 template ``stmt`` instantiates, else None. + + Only canonical-template statements are REPLACE_STATEMENT-editable, so the + inverse edit (restore the old template) is always expressible. + """ + if stmt.has_list: + return None + rest = stmt.rest.strip() + for index, (comp, args) in enumerate(V05_TEMPLATES): + if stmt.comp == comp and rest == args: + return index + return None + _STMT_RE = re.compile(r"^(?P\w+)\s*=\s*(?P\w+)\((?P.*)\)\s*$") @@ -101,7 +147,14 @@ def render(self) -> str: def parse_statements(source: str) -> list[Statement] | None: """Structural parse of a canonical program; None when a line defies the - `name = Comp(...)` shape (those programs are skipped, never mutated).""" + `name = Comp(...)` shape (those programs are skipped, never mutated). + + V0.5 statement lines (Query/Mutation/Action/State/Resource) are owned by + the canonical grammar: the line is fragment-validated through + ``validate_output(..., kind="statement")`` rather than the regex alone. + UI statement lines keep the legacy structural split so + ``Statement.render()`` stays byte-stable for existing fixture programs. + """ statements: list[Statement] = [] for line in source.splitlines(): line = line.strip() @@ -110,6 +163,11 @@ def parse_statements(source: str) -> list[Statement] | None: match = _STMT_RE.match(line) if match is None: return None + if match.group("comp") in V05_COMPONENTS: + try: + validate_output(line, kind="statement") + except Exception: # noqa: BLE001 + return None args = match.group("args") if args.startswith("["): depth = 0 @@ -152,6 +210,7 @@ def render_statements(statements: list[Statement]) -> str: return "\n".join(stmt.render() for stmt in statements) +@lru_cache(maxsize=65536) def _is_valid(source: str) -> bool: try: validate(source) @@ -162,12 +221,21 @@ def _is_valid(source: str) -> bool: @dataclass(frozen=True) class Edit: - """One bounded edit: action + statement index + component + slot.""" + """One bounded edit: action + statement index + component + slot. + + SLM-305: ``target``/``payload`` are NEW DEFAULTED fields only, so old + pickles and comparisons keep working. ``payload`` carries the leaf + component index (INSERT_SUBTREE / REPLACE_SUBTREE) or the canonical V0.5 + template index (INSERT_STATEMENT / REPLACE_STATEMENT); ``target`` is + reserved for secondary statement addressing. + """ action: int stmt: int = 0 comp: int = 0 slot: int = 0 + target: int = 0 + payload: int = 0 class TreeEditSpace: @@ -192,9 +260,38 @@ def fresh_name(self, statements: list[Statement]) -> str: return name return f"n{len(statements)}x" + def fresh_v05_name(self, statements: list[Statement], comp: str) -> str: + """Fresh V0.5 statement name with the conventional pack prefix.""" + prefix = {"Query": "q", "Mutation": "m"}.get(comp, "r") + taken = {s.name for s in statements} + for i in range(len(statements) + 8): + name = f"{prefix}{i}" + if name not in taken: + return name + return f"{prefix}{len(statements)}x" + + @staticmethod + def _placeholder(inventory: list[str], slot: int) -> str: + placeholder = inventory[slot] + if not placeholder.startswith(":"): + placeholder = f":{placeholder}" + return placeholder + def apply( - self, statements: list[Statement], edit: Edit, inventory: list[str] + self, + statements: list[Statement], + edit: Edit, + inventory: list[str], + pre_validate: Callable[[list[Statement]], bool] | None = None, ) -> list[Statement] | None: + """Apply one edit; None when inapplicable or invalid (fail closed). + + ``pre_validate`` is an optional cheap rejection hook invoked on the + mutated statement list just before parser re-validation (used by the + reachability analyzer to skip already-visited states). It can only + reject, never accept: every accepted state is still re-validated + through the real parser. + """ if edit.action == ACTION_STOP: return [Statement(**vars(s)) for s in statements] working = [ @@ -248,11 +345,188 @@ def apply( if target.name in other.children: other.children = [c for c in other.children if c != target.name] referenced = True - if not referenced: + if not referenced and target.comp not in V05_COMPONENTS: + # Unreferenced UI leaves stay immutable (old behavior); V0.5 + # pack statements are unreferenced by construction and are + # removable (inverse of INSERT_STATEMENT). return None working = [s for s in working if s.name != target.name] + elif edit.action == ACTION_ADD_CONTAINER: + # Preconditions: parent is a container, MAX_STMTS bound, comp is a + # container. The minted container starts EMPTY (leaf-only subtree) + # so REMOVE_CONTAINER is an exact safe inverse. + if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): + return None + if len(working) >= MAX_STMTS: + return None + parent = working[edit.stmt] + comp = self.components[edit.comp] + if not parent.has_list or comp not in CONTAINER_COMPONENTS: + return None + if not (0 <= edit.target < len(CONTAINER_RESTS)): + return None + name = self.fresh_name(working) + parent.children.append(name) + working.append( + Statement( + name=name, comp=comp, children=[], + rest=CONTAINER_RESTS[edit.target], has_list=True, + ) + ) + elif edit.action == ACTION_REMOVE_CONTAINER: + # Safe inverse of ADD_CONTAINER / INSERT_SUBTREE. Preconditions: + # target is a non-root container, referenced from some parent, and + # its subtree is leaf-only (no nested containers) — exactly the + # shapes the container-creating actions mint, so removal restores + # the prior state exactly. Leaf children are dropped with it. + if not (0 <= edit.stmt < len(working)): + return None + target = working[edit.stmt] + if not target.has_list or target.name == "root": + return None + by_name = {s.name: s for s in working} + if any( + by_name.get(child) is None or by_name[child].has_list + for child in target.children + ): + return None + if not any(target.name in other.children for other in working): + return None + drop = {target.name, *target.children} + working = [ + Statement( + s.name, + s.comp, + [c for c in s.children if c not in drop], + s.rest, + s.has_list, + ) + for s in working + if s.name not in drop + ] + elif edit.action == ACTION_INSERT_SUBTREE: + # Transactional declare-plus-reference: mint a small canonical + # subtree (container root + one leaf child bound to an inventory + # slot) and reference the root from an existing container, all + # re-validated as one step. Preconditions: parent is a container, + # comp is a container, payload indexes a leaf component, slot is + # in inventory, MAX_STMTS bound. Inverse: REMOVE_CONTAINER. + if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): + return None + if len(working) + 2 > MAX_STMTS: + return None + parent = working[edit.stmt] + root_comp = self.components[edit.comp] + if not parent.has_list or root_comp not in CONTAINER_COMPONENTS: + return None + if not (0 <= edit.payload < len(self.components)): + return None + leaf_comp = self.components[edit.payload] + if leaf_comp not in LEAF_COMPONENTS: + return None + if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): + return None + if not (0 <= edit.target < len(CONTAINER_RESTS)): + return None + placeholder = self._placeholder(inventory, edit.slot) + cname = self.fresh_name(working) + lname = self.fresh_name( + [*working, Statement(cname, root_comp, [], CONTAINER_REST, True)] + ) + parent.children.append(cname) + working.append( + Statement(cname, root_comp, [lname], CONTAINER_RESTS[edit.target], True) + ) + working.append( + Statement( + lname, leaf_comp, [], + json.dumps(placeholder, ensure_ascii=False), False, + ) + ) + elif edit.action == ACTION_REPLACE_SUBTREE: + # Replace the leaf of a canonical one-leaf container subtree + # (same root kind, so the subtree shape is preserved). + # Preconditions: target container has exactly one child which is a + # leaf, payload indexes a leaf component, slot is in inventory. + # The small-canonical-subtree precondition keeps the inverse + # (restore old leaf comp + slot) expressible as the same action. + if not (0 <= edit.stmt < len(working)): + return None + target = working[edit.stmt] + if not target.has_list or len(target.children) != 1: + return None + by_name = {s.name: s for s in working} + leaf = by_name.get(target.children[0]) + if leaf is None or leaf.has_list: + return None + if not (0 <= edit.payload < len(self.components)): + return None + leaf_comp = self.components[edit.payload] + if leaf_comp not in LEAF_COMPONENTS: + return None + if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): + return None + placeholder = self._placeholder(inventory, edit.slot) + leaf.comp = leaf_comp + leaf.rest = json.dumps(placeholder, ensure_ascii=False) + elif edit.action == ACTION_INSERT_STATEMENT: + # Insert a canonical V0.5 state/query/mutation statement built + # from the structured template spec and fragment-validated through + # the canonical grammar (never regex string surgery). + # Preconditions: MAX_STMTS bound, payload indexes V05_TEMPLATES. + # Inverse: REMOVE (pack statements are unreferenced). + if len(working) >= MAX_STMTS: + return None + if not (0 <= edit.payload < len(V05_TEMPLATES)): + return None + comp, args = V05_TEMPLATES[edit.payload] + candidate = Statement(self.fresh_v05_name(working, comp), comp, [], args, False) + try: + validate_output(candidate.render(), kind="statement") + except Exception: # noqa: BLE001 + return None + working.append(candidate) + elif edit.action == ACTION_REPLACE_STATEMENT: + # Swap one canonical V0.5 statement for another template. + # Preconditions: target instantiates a known canonical template + # (so the inverse — restore the old template — is expressible), + # payload indexes V05_TEMPLATES, and the swap is a real change. + if not (0 <= edit.stmt < len(working)): + return None + target = working[edit.stmt] + if v05_template_index(target) is None: + return None + if not (0 <= edit.payload < len(V05_TEMPLATES)): + return None + comp, args = V05_TEMPLATES[edit.payload] + if target.comp == comp and target.rest.strip() == args: + return None + candidate = Statement(target.name, comp, [], args, False) + try: + validate_output(candidate.render(), kind="statement") + except Exception: # noqa: BLE001 + return None + working[edit.stmt] = candidate + elif edit.action == ACTION_BIND_PLACEHOLDER: + # Transactional declaration-plus-reference: (re)bind a leaf's slot + # to an inventory placeholder. Preconditions: target is a UI leaf + # (non-root, non-container, leaf component), slot in inventory. + # Inverse: BIND_PLACEHOLDER with the old slot index. + if not (0 <= edit.stmt < len(working)): + return None + target = working[edit.stmt] + if target.has_list or target.name == "root": + return None + if target.comp not in LEAF_COMPONENTS: + return None + if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): + return None + placeholder = self._placeholder(inventory, edit.slot) + target.rest = json.dumps(placeholder, ensure_ascii=False) else: return None + if pre_validate is not None and not pre_validate(working): + return None rendered = render_statements(working) if not _is_valid(rendered): return None @@ -267,7 +541,20 @@ def sample_mutation( """One random validity-preserving mutation and the *inverse* edit (the supervised repair step) — Kapur's forward process.""" for _ in range(12): - kind = rng.choice((ACTION_REPLACE, ACTION_ADD, ACTION_REMOVE)) + kind = rng.choice( + ( + ACTION_REPLACE, + ACTION_ADD, + ACTION_REMOVE, + ACTION_ADD_CONTAINER, + ACTION_REMOVE_CONTAINER, + ACTION_INSERT_SUBTREE, + ACTION_REPLACE_SUBTREE, + ACTION_INSERT_STATEMENT, + ACTION_REPLACE_STATEMENT, + ACTION_BIND_PLACEHOLDER, + ) + ) if kind == ACTION_REPLACE: idx = rng.randrange(len(statements)) stmt = statements[idx] @@ -304,6 +591,210 @@ def sample_mutation( continue inverse = Edit(ACTION_REMOVE, len(mutated) - 1) return mutated, inverse + if kind == ACTION_ADD_CONTAINER: + # Mutation = empty container under a container; inverse = + # REMOVE_CONTAINER (exact, the minted subtree is empty). + parents = [i for i, s in enumerate(statements) if s.has_list] + if not parents or len(statements) >= MAX_STMTS: + continue + parent_idx = rng.choice(parents) + comp = rng.choice( + [c for c in CONTAINER_COMPONENTS if c in self.comp_index] + ) + mutation = Edit( + ACTION_ADD_CONTAINER, parent_idx, self.comp_index[comp], + target=rng.randrange(len(CONTAINER_RESTS)), + ) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_REMOVE_CONTAINER, len(mutated) - 1) + return mutated, inverse + if kind == ACTION_REMOVE_CONTAINER: + # Mutation = remove an empty/leaf-only container subtree; + # inverse = ADD_CONTAINER (empty) or INSERT_SUBTREE (one leaf). + by_name = {s.name: s for s in statements} + removable = [ + i + for i, s in enumerate(statements) + if s.has_list + and s.name != "root" + and any(s.name in o.children for o in statements) + and all( + by_name.get(c) is not None and not by_name[c].has_list + for c in s.children + ) + ] + if not removable: + continue + idx = rng.choice(removable) + victim = statements[idx] + if victim.comp not in self.comp_index: + continue + if victim.rest not in CONTAINER_RESTS: + # The inverse (re-minting this subtree) is only expressible + # for canonically-rested containers; skip, never fake it. + continue + rest_idx = CONTAINER_RESTS.index(victim.rest) + parent_name = next( + o.name for o in statements if victim.name in o.children + ) + inverse: Edit | None = None + if not victim.children: + inverse = Edit( + ACTION_ADD_CONTAINER, 0, self.comp_index[victim.comp], + target=rest_idx, + ) + elif len(victim.children) == 1: + leaf = by_name[victim.children[0]] + slot = self._leaf_slot_index(leaf, inventory) + if leaf.comp in self.comp_index and slot is not None: + inverse = Edit( + ACTION_INSERT_SUBTREE, + 0, + self.comp_index[victim.comp], + slot, + target=rest_idx, + payload=self.comp_index[leaf.comp], + ) + if inverse is None: + continue + mutation = Edit(ACTION_REMOVE_CONTAINER, idx) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + parent_idx = next( + i for i, s in enumerate(mutated) if s.name == parent_name + ) + inverse = Edit( + inverse.action, parent_idx, inverse.comp, inverse.slot, + target=inverse.target, payload=inverse.payload, + ) + return mutated, inverse + if kind == ACTION_INSERT_SUBTREE: + # Mutation = transactional container+leaf subtree; inverse = + # REMOVE_CONTAINER of the minted root. + parents = [i for i, s in enumerate(statements) if s.has_list] + if not parents or not inventory or len(statements) + 2 > MAX_STMTS: + continue + parent_idx = rng.choice(parents) + root_comp = rng.choice( + [c for c in CONTAINER_COMPONENTS if c in self.comp_index] + ) + leaf_comp = rng.choice( + [c for c in LEAF_COMPONENTS if c in self.comp_index] + ) + slot = rng.randrange(min(len(inventory), MAX_SLOTS)) + mutation = Edit( + ACTION_INSERT_SUBTREE, + parent_idx, + self.comp_index[root_comp], + slot, + target=rng.randrange(len(CONTAINER_RESTS)), + payload=self.comp_index[leaf_comp], + ) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_REMOVE_CONTAINER, len(mutated) - 2) + return mutated, inverse + if kind == ACTION_REPLACE_SUBTREE: + # Mutation = replace the leaf of a canonical one-leaf container + # subtree; inverse = REPLACE_SUBTREE restoring old comp+slot. + by_name = {s.name: s for s in statements} + candidates = [ + i + for i, s in enumerate(statements) + if s.has_list + and len(s.children) == 1 + and by_name.get(s.children[0]) is not None + and not by_name[s.children[0]].has_list + ] + if not candidates or not inventory: + continue + idx = rng.choice(candidates) + leaf = by_name[statements[idx].children[0]] + old_slot = self._leaf_slot_index(leaf, inventory) + if leaf.comp not in self.comp_index or old_slot is None: + continue + choices = [ + c for c in LEAF_COMPONENTS if c in self.comp_index + ] + slot = rng.randrange(min(len(inventory), MAX_SLOTS)) + new_comp = rng.choice(choices) + if new_comp == leaf.comp and slot == old_slot: + continue + mutation = Edit( + ACTION_REPLACE_SUBTREE, idx, slot=slot, + payload=self.comp_index[new_comp], + ) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit( + ACTION_REPLACE_SUBTREE, idx, slot=old_slot, + payload=self.comp_index[leaf.comp], + ) + return mutated, inverse + if kind == ACTION_INSERT_STATEMENT: + # Mutation = canonical V0.5 statement; inverse = REMOVE it. + if len(statements) >= MAX_STMTS: + continue + payload = rng.randrange(len(V05_TEMPLATES)) + mutation = Edit(ACTION_INSERT_STATEMENT, payload=payload) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_REMOVE, len(mutated) - 1) + return mutated, inverse + if kind == ACTION_REPLACE_STATEMENT: + # Mutation = swap canonical V0.5 template; inverse = swap back. + candidates = [ + i for i, s in enumerate(statements) + if v05_template_index(s) is not None + ] + if not candidates: + continue + idx = rng.choice(candidates) + old_payload = v05_template_index(statements[idx]) + assert old_payload is not None + choices = [t for t in range(len(V05_TEMPLATES)) if t != old_payload] + if not choices: + continue + mutation = Edit( + ACTION_REPLACE_STATEMENT, idx, payload=rng.choice(choices) + ) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_REPLACE_STATEMENT, idx, payload=old_payload) + return mutated, inverse + if kind == ACTION_BIND_PLACEHOLDER: + # Mutation = rebind a leaf's slot; inverse = bind the old slot. + bindable = [ + i + for i, s in enumerate(statements) + if not s.has_list + and s.name != "root" + and s.comp in LEAF_COMPONENTS + ] + if not bindable or not inventory: + continue + idx = rng.choice(bindable) + old_slot = self._leaf_slot_index(statements[idx], inventory) + if old_slot is None or len(inventory) < 2: + continue + choices = [ + s for s in range(min(len(inventory), MAX_SLOTS)) if s != old_slot + ] + if not choices: + continue + mutation = Edit(ACTION_BIND_PLACEHOLDER, idx, slot=rng.choice(choices)) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_BIND_PLACEHOLDER, idx, slot=old_slot) + return mutated, inverse # Mutation = remove a leaf; inverse = ADD it back. removable = [ i @@ -348,6 +839,20 @@ def sample_mutation( return mutated, inverse return None + def _leaf_slot_index(self, stmt: Statement, inventory: list[str]) -> int | None: + """Inventory index of the placeholder bound by a leaf, else None.""" + body = stmt.rest.strip() + if not (body.startswith('"') or body.startswith("'")): + return None + try: + literal = json.loads(body) if body.startswith('"') else body[1:-1] + except Exception: # noqa: BLE001 + return None + if isinstance(literal, str) and literal in inventory: + index = inventory.index(literal) + return index if index < MAX_SLOTS else None + return None + @dataclass class TreeEditDiffusionConfig: @@ -423,7 +928,10 @@ def forward( class TreeEditDiffusionModel(nn.Module): """Prompt-conditioned Kapur-style edit policy + value search (X22).""" - CHECKPOINT_FORMAT = 1 + # Format 2 (SLM-305): action_head grew to N_ACTIONS=11 with the extended + # edit language. Format-1 checkpoints fail closed here; warm-start them + # via ``checkpoint_migrate.migrate_tree_edit_checkpoint``. + CHECKPOINT_FORMAT = 2 def __init__( self, @@ -592,7 +1100,8 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: comp_rows = [ i for i, e in enumerate(targets) - if e.action in {ACTION_REPLACE, ACTION_ADD} + if e.action + in {ACTION_REPLACE, ACTION_ADD, ACTION_ADD_CONTAINER, ACTION_INSERT_SUBTREE} ] if comp_rows: idx = torch.tensor(comp_rows, device=device) @@ -600,7 +1109,17 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: comp_loss = F.cross_entropy(out["comp"][idx], comp_t) loss = loss + comp_loss losses["comp"] = float(comp_loss.detach().cpu()) - slot_rows = [i for i, e in enumerate(targets) if e.action == ACTION_ADD] + slot_rows = [ + i + for i, e in enumerate(targets) + if e.action + in { + ACTION_ADD, + ACTION_INSERT_SUBTREE, + ACTION_REPLACE_SUBTREE, + ACTION_BIND_PLACEHOLDER, + } + ] if slot_rows: idx = torch.tensor(slot_rows, device=device) slot_t = torch.tensor( @@ -649,6 +1168,9 @@ def _enumerate_edits( (float(action_lp[ACTION_STOP]), Edit(ACTION_STOP)) ] n_comp = comp_lp.shape[0] + leaf_comps = [ + i for i, c in enumerate(self.space.components) if c in LEAF_COMPONENTS + ] for stmt in range(min(n_stmts, MAX_STMTS)): base = float(stmt_lp[stmt]) for comp in range(n_comp): @@ -658,19 +1180,78 @@ def _enumerate_edits( Edit(ACTION_REPLACE, stmt, comp), ) ) + for rest_idx in range(len(CONTAINER_RESTS)): + scored.append( + ( + float(action_lp[ACTION_ADD_CONTAINER]) + + base + + float(comp_lp[comp]), + Edit(ACTION_ADD_CONTAINER, stmt, comp, target=rest_idx), + ) + ) for slot in range(min(n_slots, MAX_SLOTS)): + slot_score = float(slot_lp[slot]) scored.append( ( float(action_lp[ACTION_ADD]) + base + float(comp_lp[comp]) - + float(slot_lp[slot]), + + slot_score, Edit(ACTION_ADD, stmt, comp, slot), ) ) + scored.append( + ( + float(action_lp[ACTION_BIND_PLACEHOLDER]) + + base + + slot_score, + Edit(ACTION_BIND_PLACEHOLDER, stmt, slot=slot), + ) + ) + for payload in leaf_comps: + for rest_idx in range(len(CONTAINER_RESTS)): + scored.append( + ( + float(action_lp[ACTION_INSERT_SUBTREE]) + + base + + float(comp_lp[comp]) + + slot_score, + Edit(ACTION_INSERT_SUBTREE, stmt, comp, slot, + target=rest_idx, payload=payload), + ) + ) + scored.append( + ( + float(action_lp[ACTION_REPLACE_SUBTREE]) + + base + + slot_score, + Edit(ACTION_REPLACE_SUBTREE, stmt, slot=slot, + payload=payload), + ) + ) scored.append( (float(action_lp[ACTION_REMOVE]) + base, Edit(ACTION_REMOVE, stmt)) ) + scored.append( + ( + float(action_lp[ACTION_REMOVE_CONTAINER]) + base, + Edit(ACTION_REMOVE_CONTAINER, stmt), + ) + ) + for payload in range(len(V05_TEMPLATES)): + scored.append( + ( + float(action_lp[ACTION_REPLACE_STATEMENT]) + base, + Edit(ACTION_REPLACE_STATEMENT, stmt, payload=payload), + ) + ) + for payload in range(len(V05_TEMPLATES)): + scored.append( + ( + float(action_lp[ACTION_INSERT_STATEMENT]), + Edit(ACTION_INSERT_STATEMENT, payload=payload), + ) + ) scored.sort(key=lambda pair: pair[0], reverse=True) return scored @@ -854,6 +1435,14 @@ def from_checkpoint( raise ValueError( f"checkpoint kind {payload.get('kind')!r} is not tree_edit_diffusion" ) + format_version = int(payload.get("format_version") or 1) + if format_version != cls.CHECKPOINT_FORMAT: + raise ValueError( + f"tree_edit_diffusion checkpoint format_version={format_version} " + f"is not supported (expected {cls.CHECKPOINT_FORMAT}); warm-start " + "older checkpoints via " + "slm_training.models.checkpoint_migrate.migrate_tree_edit_checkpoint" + ) tokenizer = OpenUITokenizer.load(path.with_suffix(".tokenizer.json")) config = TreeEditDiffusionConfig(**payload["config"]) model = cls(tokenizer, config=config, device=device) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 45fadec38..01d77c873 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -665,7 +665,7 @@ ] }, "evals.agentv": { - "version": "v2", + "version": "v3", "kind": "harness", "paths": [ "src/slm_training/evals/agentv.py", @@ -673,6 +673,11 @@ "tests/test_evals/test_agentv.py" ], "history": [ + { + "version": "v3", + "date": "2026-07-24", + "note": "Forward valid W3C run traces to the pinned AgentV runner for summary-only observational telemetry" + }, { "version": "v2", "date": "2026-07-24", @@ -696,6 +701,11 @@ "tests/test_scripts/test_run_cap2_operator_eval_fixture.py" ], "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: document opt-in LangSmith telemetry configuration in the shared README without changing the SLM-228 disposition" + }, { "version": "v1", "date": "2026-07-23", @@ -4540,7 +4550,7 @@ ] }, "harness.model_build.eval": { - "version": "v53", + "version": "v54", "kind": "harness", "paths": [ "src/slm_training/harnesses/model_build/debt_router.py", @@ -4551,6 +4561,11 @@ "scripts/evaluate_model.py" ], "history": [ + { + "version": "v54", + "date": "2026-07-24", + "note": "Add bounded opt-in LangSmith aggregate telemetry to the shared evaluation trace while keeping local evidence authoritative" + }, { "version": "v53", "date": "2026-07-24", @@ -6188,6 +6203,11 @@ "docs/design/iter-slm242-twotower-numeric-gates-20260721.md" ], "history": [ + { + "version": "v234", + "date": "2026-07-24", + "note": "no-bump: add tree-edit checkpoint format migration without changing TwoTower behavior" + }, { "version": "v234", "date": "2026-07-24", diff --git a/src/slm_training/runtime/telemetry/trace.py b/src/slm_training/runtime/telemetry/trace.py index 76ecce574..34f74ef07 100644 --- a/src/slm_training/runtime/telemetry/trace.py +++ b/src/slm_training/runtime/telemetry/trace.py @@ -9,6 +9,8 @@ import time import urllib.request import uuid +import warnings +from datetime import datetime, timezone from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -16,6 +18,157 @@ _CURRENT: contextvars.ContextVar["RunTrace | None"] = contextvars.ContextVar( "slm_run_trace", default=None ) +_ENV_PATH = Path(__file__).resolve().parents[4] / ".env" + + +def _enabled(name: str) -> bool: + return os.getenv(name, "").strip().lower() in {"1", "true", "yes", "on"} + + +def _key_value_env(name: str) -> dict[str, str]: + values: dict[str, str] = {} + for pair in os.getenv(name, "").split(","): + key, sep, value = pair.partition("=") + if sep and key.strip(): + values[key.strip()] = value.strip() + return values + + +def _timeout_seconds(name: str, default: float, maximum: float = 5.0) -> float: + try: + return min(maximum, max(0.1, float(os.getenv(name, default)))) + except ValueError: + return default + + +def _load_local_env() -> None: + """Load the repository's ignored local configuration without overriding CI.""" + try: + from dotenv import load_dotenv + except ImportError: + return + load_dotenv(_ENV_PATH, override=False) + + +def _langsmith_api_key() -> str | None: + value = os.getenv("LANGSMITH_API_KEY", "").strip() + if not value or value == "replace_with_rotated_langsmith_key": + return None + return value + + +def _langsmith_flush_seconds() -> float: + return _timeout_seconds("SLM_LANGSMITH_FLUSH_SECONDS", 2.0) + + +def _otlp_timeout_seconds() -> float: + """Convert the standard OTLP millisecond setting to a bounded timeout.""" + try: + value = float(os.getenv("OTEL_EXPORTER_OTLP_TIMEOUT", "1000")) / 1000 + except ValueError: + value = 1.0 + return min(5.0, max(0.1, value)) + + +class _LangSmithTrace: + """Best-effort summary exporter; local OTLP remains the source of truth.""" + + def __init__(self, trace: "RunTrace") -> None: + _load_local_env() + self.trace = trace + self.client: Any | None = None + self.error: str | None = None + api_key = _langsmith_api_key() + self.config = { + "enabled": bool(api_key) and _enabled("LANGSMITH_TRACING"), + "api_key_configured": bool(api_key), + "project": os.getenv("LANGSMITH_PROJECT", "slm-training"), + "endpoint": os.getenv("LANGSMITH_ENDPOINT") or None, + "workspace_id_configured": bool(os.getenv("LANGSMITH_WORKSPACE_ID")), + } + self.run_id = uuid.UUID(hex=trace.trace_id) + + def start(self) -> None: + if not self.config["enabled"]: + return + try: + from langsmith import Client + + self.client = Client( + api_key=_langsmith_api_key(), + api_url=self.config["endpoint"], + workspace_id=os.getenv("LANGSMITH_WORKSPACE_ID") or None, + omit_traced_runtime_info=True, + ) + self.client.create_run( + id=self.run_id, + project_name=self.config["project"], + name=f"slm.{self.trace.operation}", + run_type="chain", + inputs={"run_id": self.trace.run_id, "operation": self.trace.operation}, + start_time=datetime.fromtimestamp(self.trace.start_ns / 1e9, tz=timezone.utc), + extra={ + "metadata": { + "w3c_trace_id": self.trace.trace_id, + "service": "slm-training", + **self.trace.attributes, + } + }, + ) + except Exception as exc: # noqa: BLE001 - observability must never stop a run + self.client = None + self._failed(exc) + + def summary( + self, + name: str, + *, + inputs: dict[str, Any], + outputs: dict[str, Any], + metadata: dict[str, Any], + ) -> None: + if self.client is None: + return + try: + now = datetime.now(timezone.utc) + self.client.create_run( + id=uuid.uuid4(), + trace_id=self.run_id, + parent_run_id=self.run_id, + project_name=self.config["project"], + name=name, + run_type="tool", + inputs=inputs, + outputs=outputs, + start_time=now, + end_time=now, + extra={"metadata": {"w3c_trace_id": self.trace.trace_id, **metadata}}, + ) + except Exception as exc: # noqa: BLE001 - observability must never stop a run + self._failed(exc) + + def finish(self, error: BaseException | None) -> None: + if self.client is None: + return + try: + self.client.update_run( + self.run_id, + end_time=datetime.now(timezone.utc), + outputs={"status": "failed" if error else "completed"}, + error="run failed; inspect local trace" if error else None, + ) + self.client.flush(timeout=_langsmith_flush_seconds()) + except Exception as exc: # noqa: BLE001 - observability must never stop a run + self._failed(exc) + + def manifest(self) -> dict[str, Any]: + return {**self.config, "trace_id": self.trace.trace_id, "last_export_error": self.error} + + def _failed(self, exc: Exception) -> None: + self.error = type(exc).__name__ + warnings.warn( + f"LangSmith export failed: {self.error}", RuntimeWarning, stacklevel=3 + ) def _hex_id(size: int) -> str: @@ -53,17 +206,16 @@ def _endpoint(signal: str) -> str | None: base = specific or os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") or _first_peer() if not base: return None - return base if specific else f"{base.rstrip('/')}/v1/{signal}" + if specific: + return base + normalized = base.rstrip("/") + suffix = f"/v1/{signal}" + return normalized if normalized.endswith(suffix) else f"{normalized}{suffix}" def _headers() -> dict[str, str]: - raw = os.getenv("OTEL_EXPORTER_OTLP_HEADERS") - if raw: - headers: dict[str, str] = {} - for pair in raw.split(","): - key, sep, value = pair.partition("=") - if sep and key.strip(): - headers[key.strip()] = value.strip() + headers = _key_value_env("OTEL_EXPORTER_OTLP_HEADERS") + if headers: return headers token = os.getenv("SLM_OTEL_TOKEN") if not token and os.getenv("SLM_OTEL_AUTH", "").strip().lower() == "hf": @@ -100,6 +252,7 @@ def __post_init__(self) -> None: self.bundle = self.trace_root / self.trace_id self.instance_id = str(uuid.uuid4()) self._token = None + self._langsmith = _LangSmithTrace(self) self._write_manifest() if reference: reference.parent.mkdir(parents=True, exist_ok=True) @@ -125,6 +278,7 @@ def traceparent(self) -> str: def __enter__(self) -> "RunTrace": self._token = _CURRENT.set(self) + self._langsmith.start() self.log("run.started", attributes={"slm.operation": self.operation}) return self @@ -138,9 +292,22 @@ def __exit__(self, exc_type, exc, traceback) -> None: payload = self._trace_payload(time.time_ns(), status, str(exc) if exc else "") self._append("traces", payload) self._mirror("traces", payload) + self._langsmith.finish(exc) + self._write_manifest() if self._token is not None: _CURRENT.reset(self._token) + def record_summary( + self, + name: str, + *, + inputs: dict[str, Any], + outputs: dict[str, Any], + metadata: dict[str, Any], + ) -> None: + """Export a caller-curated aggregate only; never pass raw samples here.""" + self._langsmith.summary(name, inputs=inputs, outputs=outputs, metadata=metadata) + def domain_path(self, kind: str, name: str = "records.jsonl") -> Path: if not kind.replace("_", "").replace("-", "").isalnum(): raise ValueError(f"unsafe domain trace kind: {kind!r}") @@ -213,11 +380,17 @@ def _common_attributes(self) -> dict[str, Any]: return {"slm.run.id": self.run_id, "slm.operation": self.operation, **self.attributes} def _resource_attributes(self) -> dict[str, Any]: + configured = _key_value_env("OTEL_RESOURCE_ATTRIBUTES") return { - "service.name": "slm-training", - "service.namespace": "openui", - "service.version": "0.1.0", + "service.name": ( + os.getenv("OTEL_SERVICE_NAME") + or configured.pop("service.name", None) + or "slm-training" + ), + "service.namespace": configured.pop("service.namespace", "openui"), + "service.version": configured.pop("service.version", "0.1.0"), "service.instance.id": self.instance_id, + **configured, } def _append(self, signal: str, payload: dict[str, Any]) -> None: @@ -237,7 +410,8 @@ def _mirror(self, signal: str, payload: dict[str, Any]) -> None: method="POST", ) try: - with urllib.request.urlopen(request, timeout=2): # noqa: S310 + timeout = _otlp_timeout_seconds() + with urllib.request.urlopen(request, timeout=timeout): # noqa: S310 pass except OSError as exc: self._write_manifest(export_error=str(exc)) @@ -263,6 +437,7 @@ def _write_manifest(self, *, export_error: str | None = None) -> None: "otlp_json": True, "remote_endpoint_configured": bool(_endpoint("traces")), "last_export_error": export_error, + "langsmith": self._langsmith.manifest(), } path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") diff --git a/tests/test_evals/test_agentv.py b/tests/test_evals/test_agentv.py index 1bd9c341f..9269b6302 100644 --- a/tests/test_evals/test_agentv.py +++ b/tests/test_evals/test_agentv.py @@ -2,6 +2,7 @@ import json from pathlib import Path +from types import SimpleNamespace import slm_training.evals.agentv as agentv_module @@ -132,6 +133,39 @@ def test_agentv_contract_checks_fail_even_when_pass_flag_is_true(tmp_path) -> No assert published["criteria"]["failed"] == 1 +def test_agentv_forwards_w3c_trace_id_to_the_node_runner(tmp_path, monkeypatch) -> None: + runner = tmp_path / "runner.mjs" + runner.write_text("// fixture") + sdk_root = tmp_path / "sdk-root" + captured = {} + (tmp_path / "trace.json").write_text( + json.dumps({"trace_id": "0123456789abcdef0123456789abcdef"}) + ) + monkeypatch.setattr(agentv_module, "_agentv_runtime", lambda _: (runner, sdk_root)) + + def fake_run(command, **kwargs): + captured["command"] = command + return SimpleNamespace( + returncode=0, + stdout=json.dumps({"summary": {}, "artifacts": {}}), + stderr="", + ) + + monkeypatch.setattr(agentv_module.subprocess, "run", fake_run) + publish_agentv_evaluation( + tmp_path, + name="trace-link", + claim="fixture_wiring_not_ship", + cases=[{"id": "case", "criteria": "passes", "pass": True}], + ) + assert captured["command"][-4:] == [ + "--trace-id", + "0123456789abcdef0123456789abcdef", + "--run-id", + tmp_path.name, + ] + + def test_agentv_model_bundle_cannot_pass_a_smoke_only_run(tmp_path) -> None: published = publish_model_evaluation( tmp_path, diff --git a/tests/test_harnesses/experiments/test_slm299_edit_reachability.py b/tests/test_harnesses/experiments/test_slm299_edit_reachability.py index 639f5af03..9d9aba6c9 100644 --- a/tests/test_harnesses/experiments/test_slm299_edit_reachability.py +++ b/tests/test_harnesses/experiments/test_slm299_edit_reachability.py @@ -55,7 +55,9 @@ def test_reachable_leaf_replacement_from_seeded_state() -> None: ) assert case.verdict is Verdict.PROVEN_REACHABLE assert case.edit_lower_bound == 1 - assert case.path[0]["action"] == "REPLACE" + # Either the v1 REPLACE or the extended REPLACE_SUBTREE is a valid + # 1-edit proof here; BFS returns the first in deterministic order. + assert case.path[0]["action"] in {"REPLACE", "REPLACE_SUBTREE"} def test_seed_identity_is_zero_edits() -> None: @@ -71,12 +73,26 @@ def test_multi_container_target_needs_container_add() -> None: SEED, 'root = Stack([card], "column")\ntitle = TextContent(":t")\ncard = Card([title], "column")', slot_inventory=[":t"], + mode="v1", ) assert case.verdict is Verdict.PROVEN_UNREACHABLE assert case.reason_code == "needs_container_add" assert case.edit_lower_bound is None +def test_multi_container_target_reachable_in_extended_space() -> None: + # SLM-305: the real ADD_CONTAINER / BIND_PLACEHOLDER actions close the + # needs_container_add gap; the synthetic lane is retired for this space. + case = analyze_reachability( + SEED, + 'root = Stack([card], "column")\ntitle = TextContent(":t")\ncard = Card([title], "column")', + slot_inventory=[":t"], + ) + assert case.verdict is Verdict.PROVEN_REACHABLE + actions = [step["action"] for step in case.path] + assert "ADD_CONTAINER" in actions or "INSERT_SUBTREE" in actions + + # (c) state/query/action (V0.5) program forms ------------------------------ @pytest.mark.parametrize( @@ -108,17 +124,32 @@ def test_pack_feature_targets_are_unsupported(target: str) -> None: def test_tiny_budget_yields_unknown_not_unreachable() -> None: case = analyze_reachability( - SEED, _FIVE_LEAF_TARGET, slot_inventory=[":x"], max_edits=2 + SEED, _FIVE_LEAF_TARGET, slot_inventory=[":x"], max_edits=2, mode="v1" ) assert case.verdict is Verdict.UNKNOWN_BUDGET assert case.reason_code == "budget" # And it is genuinely reachable with enough budget — the UNKNOWN was a - # budget artifact, never a proof of unreachability. - full = analyze_reachability(SEED, _FIVE_LEAF_TARGET, slot_inventory=[":x"]) + # budget artifact, never a proof of unreachability (v1 space keeps the + # branching small enough to certify the depth-5 bound). + full = analyze_reachability( + SEED, _FIVE_LEAF_TARGET, slot_inventory=[":x"], mode="v1" + ) assert full.verdict is Verdict.PROVEN_REACHABLE assert full.edit_lower_bound == 5 +def test_extended_budget_stop_is_explicit_unknown_never_unreachable() -> None: + # The extended space's branching means deep targets exhaust the node + # budget: the verdict must be UNKNOWN_BUDGET, never a silent invariant or + # a false unreachability proof. + case = analyze_reachability( + SEED, _FIVE_LEAF_TARGET, slot_inventory=[":x"], node_budget=40 + ) + assert case.verdict is Verdict.UNKNOWN_BUDGET + assert case.reason_code == "budget" + assert case.details["stop"] == "node_budget" + + def test_unknowns_are_excluded_from_reachable_fraction() -> None: cases = [ ("hit", analyze_reachability(SEED, SEED, slot_inventory=[":x"])), @@ -142,7 +173,7 @@ def test_unknowns_are_excluded_from_reachable_fraction() -> None: def test_synthetic_add_container_flips_verdict_and_is_marked() -> None: - real = analyze_reachability(SEED, _CARD_TARGET, slot_inventory=[":x"]) + real = analyze_reachability(SEED, _CARD_TARGET, slot_inventory=[":x"], mode="v1") assert real.verdict is Verdict.PROVEN_UNREACHABLE assert real.reason_code == "needs_container_add" @@ -151,6 +182,7 @@ def test_synthetic_add_container_flips_verdict_and_is_marked() -> None: _CARD_TARGET, slot_inventory=[":x"], extra_actions=[add_container_action()], + mode="v1", ) assert what_if.verdict is Verdict.PROVEN_REACHABLE assert what_if.details["uses_synthetic_actions"] is True @@ -175,7 +207,7 @@ def _random_statements(rng: random.Random) -> list: def test_transition_parity_with_tree_edit_space() -> None: # The analyzer must enumerate exactly the children TreeEditSpace.apply - # admits — no more, no fewer — on random reachable states. + # admits — no more, no fewer — on random reachable states (v1 space). from slm_training.harnesses.experiments.slm299_edit_reachability import ( _enumerate_children, ) @@ -187,7 +219,7 @@ def test_transition_parity_with_tree_edit_space() -> None: statements = _random_statements(rng) via_analyzer = { render_statements(child) - for child, _ in _enumerate_children(space, statements, inventory) + for child, _ in _enumerate_children(space, statements, inventory, mode="v1") } via_reference: set[str] = set() n_comp = len(space.components) @@ -208,6 +240,47 @@ def test_transition_parity_with_tree_edit_space() -> None: assert via_analyzer == via_reference +def test_extended_enumeration_is_superset_and_deterministic() -> None: + # Extended mode enumerates every v1 child plus the new real actions, in a + # deterministic order (same state → same enumeration). + from slm_training.harnesses.experiments.slm299_edit_reachability import ( + _enumerate_children, + ) + + space = TreeEditSpace() + inventory = [":x", ":y"] + statements = parse_statements(SEED) + assert statements is not None + v1 = _enumerate_children(space, statements, inventory, mode="v1") + ext_first = _enumerate_children(space, statements, inventory, mode="extended") + ext_second = _enumerate_children(space, statements, inventory, mode="extended") + assert [a for _, a in ext_first] == [a for _, a in ext_second] + assert [ + render_statements(child) for child, _ in ext_first + ] == [render_statements(child) for child, _ in ext_second] + v1_keys = {render_statements(child) for child, _ in v1} + ext_keys = {render_statements(child) for child, _ in ext_first} + assert v1_keys <= ext_keys + ext_actions = {action["action"] for _, action in ext_first} + assert {"ADD_CONTAINER", "INSERT_STATEMENT", "INSERT_SUBTREE"} <= ext_actions + + +def test_v05_canonical_template_target_is_reachable_in_extended_space() -> None: + target = ( + 'root = Stack([], "column")\n' + 'q0 = Query("tool", {arg: $x}, {default: []}, 15)' + ) + extended = analyze_reachability(SEED, target, slot_inventory=[":x"]) + assert extended.verdict is Verdict.PROVEN_REACHABLE + assert any(step["action"] == "INSERT_STATEMENT" for step in extended.path) + v1 = analyze_reachability(SEED, target, slot_inventory=[":x"], mode="v1") + assert v1.verdict is Verdict.PROVEN_UNREACHABLE + # v1 has no statement actions at all: the pack line is either flagged as + # an unsupported pack feature (marker name prefix) or as a component the + # UI statement space cannot produce. + assert v1.reason_code in {"unsupported_pack_feature", "unsupported_component"} + + # (g) ship-gate integration ------------------------------------------------- def _gate_suites() -> dict[str, dict[str, object]]: @@ -286,10 +359,12 @@ def _mini_corpora() -> dict[str, list[dict[str, object]]]: def test_audit_report_is_deterministic() -> None: first = build_report( - _mini_corpora(), max_edits=8, node_budget=200, generated_at="2026-07-24T00:00:00Z" + _mini_corpora(), max_edits=8, node_budget=200, generated_at="2026-07-24T00:00:00Z", + mode="v1", ) second = build_report( - _mini_corpora(), max_edits=8, node_budget=200, generated_at="2026-07-24T00:00:00Z" + _mini_corpora(), max_edits=8, node_budget=200, generated_at="2026-07-24T00:00:00Z", + mode="v1", ) first.pop("version_stamp") second.pop("version_stamp") @@ -298,3 +373,18 @@ def test_audit_report_is_deterministic() -> None: assert first["suites"]["smoke"]["reachable_fraction"] == 0.5 assert first["suites"]["ood"]["status"] == "corpus_unavailable" assert first["x22_evidence_annotations"] + + +def test_audit_compare_reports_old_vs_extended() -> None: + payload = build_report( + _mini_corpora(), max_edits=8, node_budget=200, + generated_at="2026-07-24T00:00:00Z", mode="extended", compare=True, + ) + assert payload["suites"]["smoke"]["reachable_fraction"] == 1.0 + assert payload["suites_v1"]["smoke"]["reachable_fraction"] == 0.5 + comp = payload["old_vs_extended"]["smoke"] + assert comp["n_verdict_flips"] == 1 + assert comp["verdict_flips"][0]["id"] == "miss" + assert "version_stamp" in payload + md = render_markdown(payload) + assert "Old (v1) vs extended" in md diff --git a/tests/test_harnesses/model_build/test_eval_gates.py b/tests/test_harnesses/model_build/test_eval_gates.py index 37472d318..ce08fe2c7 100644 --- a/tests/test_harnesses/model_build/test_eval_gates.py +++ b/tests/test_harnesses/model_build/test_eval_gates.py @@ -23,6 +23,7 @@ from slm_training.harnesses.model_build.eval_runner import ( _effective_evaluation_policy, _is_meaningful_program, + _record_langsmith_evaluation, component_type_recall, evaluate, evaluate_suites, @@ -38,6 +39,44 @@ from slm_training.models.decode_stats import DecodeStats +def test_langsmith_evaluation_summary_excludes_per_example_details( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured = {} + + class Trace: + def record_summary(self, name, **kwargs) -> None: + captured["name"] = name + captured.update(kwargs) + + monkeypatch.setattr( + "slm_training.runtime.telemetry.current_trace", lambda: Trace() + ) + _record_langsmith_evaluation( + SimpleNamespace(run_id="safe-summary"), + suites={ + "smoke": { + "n": 1, + "parse_rate": 1.0, + "details": [{"prompt": "do not export", "completion": "secret"}], + } + }, + scoreboard={ + "run_class": "scratch_matrix", + "checkpoint_sha256": "a" * 64, + "eval_data_manifest_sha": "b" * 64, + "code_git_sha": "c" * 40, + "version_stamp": {"stamp_schema": "version_stamp/v1"}, + "gates": {"pass": False, "failures": ["smoke:parse_rate"]}, + "agentv": {"summary": {"passed": 0, "failed": 1}}, + }, + ) + assert captured["name"] == "evaluation.summary" + assert captured["outputs"]["suites"]["smoke"] == {"n": 1, "parse_rate": 1.0} + assert "details" not in str(captured) + assert "do not export" not in str(captured) + + def test_evaluation_policy_reports_loaded_checkpoint_settings() -> None: config = ModelBuildConfig( train_dir=Path("train"), diff --git a/tests/test_models/test_tree_edit_diffusion.py b/tests/test_models/test_tree_edit_diffusion.py index 1b1cfea4e..368bf729f 100644 --- a/tests/test_models/test_tree_edit_diffusion.py +++ b/tests/test_models/test_tree_edit_diffusion.py @@ -54,6 +54,166 @@ def test_mutations_preserve_validity_and_inverse_restores() -> None: assert restored_any +# --- SLM-305 extended edit language ----------------------------------------- + +from slm_training.models.tree_edit_diffusion import ( # noqa: E402 + ACTION_ADD_CONTAINER, + ACTION_BIND_PLACEHOLDER, + ACTION_INSERT_STATEMENT, + ACTION_INSERT_SUBTREE, + ACTION_REMOVE, + ACTION_REMOVE_CONTAINER, + ACTION_REPLACE_STATEMENT, + ACTION_REPLACE_SUBTREE, + Edit, +) + + +def test_render_is_byte_stable_for_fixture_programs() -> None: + statements = parse_statements(PROGRAM) + assert statements is not None + assert render_statements(statements) == PROGRAM + + +def test_extended_actions_apply_and_invert_exactly() -> None: + """Every new action applies only under its preconditions, stays valid, and + its supervised inverse restores the exact prior program text.""" + space = TreeEditSpace() + ci = space.comp_index + base = parse_statements(PROGRAM) + assert base is not None + + def apply(edit: Edit, stmts=None): + nxt = space.apply(stmts or base, edit, INVENTORY) + assert nxt is not None, edit + validate(render_statements(nxt)) + return nxt + + # ADD_CONTAINER <-> REMOVE_CONTAINER (empty subtree: exact safe inverse) + added = apply(Edit(ACTION_ADD_CONTAINER, 0, ci["Card"])) + assert len(added) == len(base) + 1 + assert render_statements( + apply(Edit(ACTION_REMOVE_CONTAINER, len(added) - 1), added) + ) == PROGRAM + # REMOVE_CONTAINER rejects non-empty nested subtrees (fail closed) + assert space.apply(base, Edit(ACTION_REMOVE_CONTAINER, 0), INVENTORY) is None + # INSERT_SUBTREE <-> REMOVE_CONTAINER (container + leaf child) + inserted = apply(Edit(ACTION_INSERT_SUBTREE, 0, ci["Stack"], 1, payload=ci["Button"])) + assert render_statements( + apply(Edit(ACTION_REMOVE_CONTAINER, len(inserted) - 2), inserted) + ) == PROGRAM + # REPLACE_SUBTREE swaps the single leaf of a canonical subtree, same root + replaced = apply(Edit(ACTION_REPLACE_SUBTREE, 1, slot=1, payload=ci["Image"])) + assert replaced[1].comp == "Card" # root kind preserved + assert render_statements( + apply(Edit(ACTION_REPLACE_SUBTREE, 1, slot=0, payload=ci["TextContent"]), replaced) + ) == PROGRAM + # INSERT_STATEMENT / REPLACE_STATEMENT (V0.5 canonical forms) + REMOVE + with_query = apply(Edit(ACTION_INSERT_STATEMENT, payload=0)) + assert with_query[-1].comp == "Query" + swapped = apply(Edit(ACTION_REPLACE_STATEMENT, len(with_query) - 1, payload=1), with_query) + assert swapped[-1].comp == "Mutation" + assert render_statements( + apply(Edit(ACTION_REPLACE_STATEMENT, len(swapped) - 1, payload=0), swapped) + ) == render_statements(with_query) + removed = space.apply(with_query, Edit(ACTION_REMOVE, len(with_query) - 1), INVENTORY) + assert removed is not None + assert render_statements(removed) == PROGRAM + # BIND_PLACEHOLDER rebinds a leaf slot; references (children lists) stay + bound = apply(Edit(ACTION_BIND_PLACEHOLDER, 4, slot=0)) + assert bound[1].children == base[1].children + assert render_statements( + apply(Edit(ACTION_BIND_PLACEHOLDER, 4, slot=1), bound) + ) == PROGRAM + # Out-of-precondition edits fail closed. + assert space.apply(base, Edit(ACTION_BIND_PLACEHOLDER, 0, slot=0), INVENTORY) is None + assert space.apply(base, Edit(ACTION_REPLACE_STATEMENT, 0, payload=0), INVENTORY) is None + + +def test_extended_sample_mutation_loop_restores() -> None: + space = TreeEditSpace() + rng = random.Random(11) + statements = parse_statements(PROGRAM) + assert statements is not None + seen_actions: set[int] = set() + for _ in range(80): + step = space.sample_mutation(statements, INVENTORY, rng) + if step is None: + continue + mutated, inverse = step + validate(render_statements(mutated)) + repaired = space.apply(mutated, inverse, INVENTORY) + assert repaired is not None + validate(render_statements(repaired)) + seen_actions.add(inverse.action) + # The seeded loop exercised the extended inverse-edit supervision. + assert seen_actions - {0} + + +def test_edit_new_fields_default() -> None: + old = Edit(1, 2, 3, 4) + assert old.target == 0 and old.payload == 0 + assert old == Edit(1, 2, 3, 4, 0, 0) + + +def test_checkpoint_format2_fail_closed_and_migration(tmp_path) -> None: + records = [ + ExampleRecord( + id="a", + prompt="Hero card with title, body, and a CTA button.", + openui=PROGRAM, + placeholders=INVENTORY, + ) + ] + cfg = TreeEditDiffusionConfig( + d_model=32, n_heads=4, context_layers=1, denoiser_layers=1, seed=3, + ) + model = TreeEditDiffusionModel.from_records(records, config=cfg, device="cpu") + path = tmp_path / "ckpt.pt" + model.save(path) + # Round-trip at format 2. + loaded = TreeEditDiffusionModel.from_checkpoint(path, device="cpu") + assert loaded.policy.action_head.out_features == model.policy.action_head.out_features + + # Simulate a format-1 checkpoint: shrink the action head to 4 rows. + import torch as _torch + + payload = _torch.load(path, map_location="cpu", weights_only=False) + payload["format_version"] = 1 + sd = payload["state_dict"] + old_w = sd["policy.action_head.weight"][:4].clone() + old_b = sd["policy.action_head.bias"][:4].clone() + sd["policy.action_head.weight"] = old_w + sd["policy.action_head.bias"] = old_b + old_path = tmp_path / "ckpt_v1.pt" + _torch.save(payload, old_path) + (tmp_path / "ckpt_v1.tokenizer.json").write_text( + (tmp_path / "ckpt.tokenizer.json").read_text(encoding="utf-8"), + encoding="utf-8", + ) + + # Unmigrated old checkpoint fails closed with a clear error. + with pytest.raises(ValueError, match="format_version=1"): + TreeEditDiffusionModel.from_checkpoint(old_path, device="cpu") + + # Migration warm-starts: old action rows preserved, new rows initialized. + from slm_training.models.checkpoint_migrate import migrate_tree_edit_checkpoint + + out_path = tmp_path / "ckpt_migrated.pt" + report = migrate_tree_edit_checkpoint( + source_checkpoint=old_path, output_checkpoint=out_path + ) + assert report["source_format_version"] == 1 + assert report["output_format_version"] == 2 + assert report["preserved_action_head_rows"] == 4 + assert (tmp_path / "ckpt_migrated.migrate.json").exists() + migrated = TreeEditDiffusionModel.from_checkpoint(out_path, device="cpu") + new_w = migrated.policy.action_head.weight + assert new_w.shape[0] == model.policy.action_head.weight.shape[0] + assert _torch.allclose(new_w[:4], old_w) + assert _torch.allclose(migrated.policy.action_head.bias[:4], old_b) + + def test_training_loss_decode_all_valid_and_checkpoint(tmp_path) -> None: records = [ ExampleRecord( diff --git a/tests/test_runtime_trace.py b/tests/test_runtime_trace.py index a793be0f2..6702bb21d 100644 --- a/tests/test_runtime_trace.py +++ b/tests/test_runtime_trace.py @@ -1,11 +1,23 @@ from __future__ import annotations import json +import sys +import uuid from pathlib import Path +from types import SimpleNamespace +import pytest + +import slm_training.runtime.telemetry.trace as trace_module from slm_training.runtime.telemetry import run_trace +@pytest.fixture(autouse=True) +def _disable_real_langsmith_exports(monkeypatch: pytest.MonkeyPatch) -> None: + """Focused tests use fakes when exercising the remote publisher.""" + monkeypatch.setenv("LANGSMITH_TRACING", "false") + + def test_run_trace_is_w3c_correlated_and_reused(tmp_path: Path) -> None: run_dir = tmp_path / "runs" / "demo" trace_root = tmp_path / "traces" @@ -37,6 +49,113 @@ def test_domain_trace_path_is_centralized(tmp_path: Path) -> None: assert path == tmp_path / trace.trace_id / "domain" / "synthesis" / "records.jsonl" +def test_langsmith_summary_uses_the_w3c_trace_and_safe_payload( + tmp_path: Path, monkeypatch +) -> None: + calls: list[tuple[str, tuple, dict]] = [] + + class FakeClient: + def __init__(self, **kwargs) -> None: + calls.append(("client", (), kwargs)) + + def create_run(self, **kwargs) -> None: + calls.append(("create", (), kwargs)) + + def update_run(self, *args, **kwargs) -> None: + calls.append(("update", args, kwargs)) + + def flush(self, **kwargs) -> None: + calls.append(("flush", (), kwargs)) + + monkeypatch.setitem(sys.modules, "langsmith", SimpleNamespace(Client=FakeClient)) + monkeypatch.setenv("LANGSMITH_API_KEY", "test-key") + monkeypatch.setenv("LANGSMITH_TRACING", "true") + run_dir = tmp_path / "runs" / "langsmith" + with run_trace("langsmith", "eval", run_dir=run_dir) as trace: + trace.record_summary( + "evaluation.summary", + inputs={"run_id": "langsmith", "suites": ["smoke"]}, + outputs={"suites": {"smoke": {"parse_rate": 1.0}}}, + metadata={"version_stamp": {"stamp_schema": "version_stamp/v1"}}, + ) + + creates = [payload for kind, _, payload in calls if kind == "create"] + assert len(creates) == 2 + root, child = creates + assert root["id"] == uuid.UUID(hex=trace.trace_id) + assert child["trace_id"] == root["id"] + assert child["parent_run_id"] == root["id"] + assert "prompt" not in json.dumps(child, default=str) + manifest = json.loads((tmp_path / "traces" / trace.trace_id / "manifest.json").read_text()) + assert manifest["langsmith"]["enabled"] is True + assert manifest["langsmith"]["project"] == "slm-training" + assert ("flush", (), {"timeout": 2.0}) in calls + + +def test_langsmith_loads_the_repository_env_file(tmp_path: Path, monkeypatch) -> None: + calls: list[dict] = [] + + class FakeClient: + def __init__(self, **kwargs) -> None: + calls.append(kwargs) + + def create_run(self, **kwargs) -> None: + pass + + def update_run(self, *args, **kwargs) -> None: + pass + + def flush(self, **kwargs) -> None: + pass + + env_path = tmp_path / ".env" + env_path.write_text( + "LANGSMITH_TRACING=true\n" + "LANGSMITH_API_KEY=test-key\n" + "LANGSMITH_PROJECT=env-project\n", + encoding="utf-8", + ) + for name in ("LANGSMITH_TRACING", "LANGSMITH_API_KEY", "LANGSMITH_PROJECT"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(trace_module, "_ENV_PATH", env_path) + monkeypatch.setitem(sys.modules, "langsmith", SimpleNamespace(Client=FakeClient)) + with run_trace("env", "eval", trace_root=tmp_path) as trace: + pass + assert calls[0]["api_key"] == "test-key" + manifest = json.loads((tmp_path / trace.trace_id / "manifest.json").read_text()) + assert manifest["langsmith"]["project"] == "env-project" + assert manifest["langsmith"]["api_key_configured"] is True + + +def test_langsmith_placeholder_is_inert(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("LANGSMITH_API_KEY", "replace_with_rotated_langsmith_key") + monkeypatch.setenv("LANGSMITH_TRACING", "true") + with run_trace("placeholder", "eval", trace_root=tmp_path) as trace: + pass + manifest = json.loads((tmp_path / trace.trace_id / "manifest.json").read_text()) + assert manifest["langsmith"]["enabled"] is False + assert manifest["langsmith"]["api_key_configured"] is False + + +def test_langsmith_export_failures_do_not_stop_local_tracing(tmp_path: Path, monkeypatch) -> None: + class FailingClient: + def __init__(self, **kwargs) -> None: + pass + + def create_run(self, **kwargs) -> None: + raise OSError("offline") + + monkeypatch.setitem(sys.modules, "langsmith", SimpleNamespace(Client=FailingClient)) + monkeypatch.setenv("LANGSMITH_API_KEY", "test-key") + monkeypatch.setenv("LANGSMITH_TRACING", "true") + with pytest.warns(RuntimeWarning, match="LangSmith export failed"): + with run_trace("offline", "eval", trace_root=tmp_path) as trace: + trace.log("still.local") + assert (tmp_path / trace.trace_id / "signals" / "traces").is_dir() + manifest = json.loads((tmp_path / trace.trace_id / "manifest.json").read_text()) + assert manifest["langsmith"]["last_export_error"] == "OSError" + + def test_endpoint_precedence_includes_peer_fallback(monkeypatch) -> None: from slm_training.runtime.telemetry.trace import _endpoint @@ -58,6 +177,32 @@ def test_endpoint_precedence_includes_peer_fallback(monkeypatch) -> None: assert _endpoint("logs") == "http://exact/ingest" +def test_otlp_standard_endpoint_and_resource_config_are_honored( + tmp_path: Path, monkeypatch +) -> None: + from slm_training.runtime.telemetry.trace import _endpoint, _otlp_timeout_seconds + + monkeypatch.delenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318/v1/logs/") + assert _endpoint("logs") == "http://collector:4318/v1/logs" + + monkeypatch.setenv("OTEL_SERVICE_NAME", "slm-training-ci") + monkeypatch.setenv( + "OTEL_RESOURCE_ATTRIBUTES", + "service.namespace=ci,deployment.environment=test", + ) + with run_trace("resources", "eval", trace_root=tmp_path) as trace: + attributes = trace._resource_attributes() + assert attributes["service.name"] == "slm-training-ci" + assert attributes["service.namespace"] == "ci" + assert attributes["deployment.environment"] == "test" + + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "1000") + assert _otlp_timeout_seconds() == 1.0 + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "90000") + assert _otlp_timeout_seconds() == 5.0 + + def test_mirror_headers_resolution(monkeypatch) -> None: from slm_training.runtime.telemetry.trace import _headers From 6dd710a7d13753052e3c44cef8ae3d311f05e467 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:51:20 -0500 Subject: [PATCH 04/18] SLM-305: extend tree edit language --- src/slm_training/resources/versions.json | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 01d77c873..0397d95b9 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -4475,7 +4475,7 @@ ] }, "harness.experiments.slm299_edit_reachability": { - "version": "v1", + "version": "v2", "kind": "harness", "paths": [ "src/slm_training/harnesses/experiments/slm299_edit_reachability.py", @@ -4483,9 +4483,17 @@ "tests/test_harnesses/experiments/test_slm299_edit_reachability.py", "docs/design/iter-slm299-edit-reachability-20260724.json", "docs/design/iter-slm299-edit-reachability-20260724.md", - "src/slm_training/models/tree_edit_diffusion.py" + "src/slm_training/models/tree_edit_diffusion.py", + "docs/design/iter-slm305-edit-language-20260724.json", + "docs/design/iter-slm305-edit-language-20260724.md", + "tests/test_models/test_tree_edit_diffusion.py" ], "history": [ + { + "version": "v2", + "date": "2026-07-24", + "note": "SLM-305: extend the valid-state edit language (ADD_CONTAINER/REMOVE_CONTAINER, INSERT/REPLACE_SUBTREE, V0.5 statement edits, BIND_PLACEHOLDER) with inverses, parser-backed validation, checkpoint format 2 + migration, and extended reachability mode" + }, { "version": "v1", "date": "2026-07-24", From ae5448c594233b304ea22720604a962ad041fe63 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:44:33 -0500 Subject: [PATCH 05/18] SLM-308: exact bounded distance value labels + progress calibration - slm308_distance_oracle.py: bounded reverse-BFS over canonical AST fingerprints via the real extended 11-action transitions; EXACT / BOUNDED / UNKNOWN labels (budget never conflated with distance); cache keyed by action-schema version, grammar sha, inventory, target and state hashes. - tree_edit_diffusion: value_label_mode (bounded_distance default for new configs; pre-field checkpoints inject mutation_count for behavior parity, format stays 2); normalized oracle cost-to-go with UNKNOWN masked out of MSE; pairwise progress margin loss (independently tested); decode paths never read gold distance (audit test). - Matched fixture experiment (mutation_count vs bounded_distance, identical budgets, near-gold + seed-trajectory states): rank corr 0.433->0.505 (+0.072 < 0.10), beam regret 0.889->0.889 (+0.0 < 0.05), Brier 0.031->0.004, UNKNOWN coverage 0.175. Preregistered thresholds written before results; verdict honestly rejected at this budget. - 16 new tests; slm308 component v1; slm299 component v3. --- .../iter-slm308-distance-value-20260724.json | 150 +++++ .../iter-slm308-distance-value-20260724.md | 33 + package-lock.json | 110 --- package.json | 4 - scripts/run_agentv_eval.mjs | 48 +- scripts/run_slm308_distance_value.py | 627 ++++++++++++++++++ .../experiments/slm308_distance_oracle.py | 398 +++++++++++ .../models/tree_edit_diffusion.py | 144 +++- src/slm_training/resources/versions.json | 32 +- .../experiments/test_slm308_distance_value.py | 357 ++++++++++ 10 files changed, 1757 insertions(+), 146 deletions(-) create mode 100644 docs/design/iter-slm308-distance-value-20260724.json create mode 100644 docs/design/iter-slm308-distance-value-20260724.md create mode 100644 scripts/run_slm308_distance_value.py create mode 100644 src/slm_training/harnesses/experiments/slm308_distance_oracle.py create mode 100644 tests/test_harnesses/experiments/test_slm308_distance_value.py diff --git a/docs/design/iter-slm308-distance-value-20260724.json b/docs/design/iter-slm308-distance-value-20260724.json new file mode 100644 index 000000000..c0ca56e41 --- /dev/null +++ b/docs/design/iter-slm308-distance-value-20260724.json @@ -0,0 +1,150 @@ +{ + "experiment": "slm308-distance-value", + "issue": "SLM-308", + "question": "Do bounded-distance oracle value labels beat mutation-count value labels at equal fixture budget (rank correlation + beam regret)?", + "preregistered_thresholds": { + "beam_regret_improvement_min": 0.05, + "rank_correlation_improvement_min": 0.1, + "direction": "arm_b(bounded_distance) minus arm_a(mutation_count)", + "verdict_rule": "adopted iff both thresholds are met, else rejected" + }, + "config": { + "steps": 8, + "batch_size": 4, + "seed": 0, + "n_records": 8, + "eval_max_depth": 8, + "eval_node_budget": 8 + }, + "oracle_cost": { + "n_states": 40, + "n_unknown": 7, + "unknown_coverage": 0.175, + "n_measurable": 17, + "wall_ms_total": 26468.863566005894, + "ms_per_state": 661.7215891501473, + "cache": { + "label_hits": 0, + "label_misses": 40, + "map_builds": 8, + "target_maps": 8, + "labels": 40, + "map_nodes": 64 + } + }, + "arms": { + "arm_a": { + "value_label_mode": "mutation_count", + "metrics": { + "n_measurable": 17, + "rank_correlation": 0.4330127018922193, + "concordance": 0.75, + "brier": 0.030793564056903686, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 8, + "mean_predicted_value": 0.6458271443843842, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 9, + "mean_predicted_value": 0.6430932150946723, + "mean_target": 0.75 + } + ], + "beam_regret_mean": 0.8888888888888888, + "beam_regret_n": 9, + "beam_regret_excluded": 8, + "by_origin": { + "near_gold": { + "n": 14, + "rank_correlation": 0.35805743701971643 + }, + "seed_trajectory": { + "n": 3, + "rank_correlation": null + } + } + }, + "training_metrics": { + "action": 1.8480353355407715, + "stmt": 2.9644241333007812, + "slot": 0.3795056641101837, + "value": 0.08967626839876175, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + }, + "arm_b": { + "value_label_mode": "bounded_distance", + "metrics": { + "n_measurable": 17, + "rank_correlation": 0.5051814855409226, + "concordance": 0.7916666666666666, + "brier": 0.004281158882868815, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 8, + "mean_predicted_value": 0.7873154133558273, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 9, + "mean_predicted_value": 0.7852203448613485, + "mean_target": 0.75 + } + ], + "beam_regret_mean": 0.8888888888888888, + "beam_regret_n": 9, + "beam_regret_excluded": 8, + "by_origin": { + "near_gold": { + "n": 14, + "rank_correlation": 0.4296689244236597 + }, + "seed_trajectory": { + "n": 3, + "rank_correlation": null + } + } + }, + "training_metrics": { + "action": 1.8418681621551514, + "stmt": 2.97049617767334, + "slot": 0.37653154134750366, + "value": 0.031629983335733414, + "pairwise_progress": 0.10255739837884903, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + }, + "improvements": { + "rank_correlation": 0.07216878364870327, + "concordance": 0.04166666666666663, + "beam_regret": 0.0, + "brier": 0.026512405174034873 + }, + "threshold_checks": { + "rank_correlation_ok": false, + "beam_regret_ok": false + }, + "verdict": "rejected", + "honesty": "Fixture-scale matched arms; UNKNOWN/unbounded states are excluded from distance-referenced metrics and counted, never coerced. A fixture verdict is wiring/label evidence, not a production ship claim.", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "6dd710a7d13753052e3c44cef8ae3d311f05e467", + "code_dirty": true, + "components": { + "harness.experiments.slm308_distance_value": "v1", + "harness.experiments.slm299_edit_reachability": "v3" + }, + "stamped_at": "2026-07-24T21:40:05.585150+00:00" + } +} diff --git a/docs/design/iter-slm308-distance-value-20260724.md b/docs/design/iter-slm308-distance-value-20260724.md new file mode 100644 index 000000000..20ca3e747 --- /dev/null +++ b/docs/design/iter-slm308-distance-value-20260724.md @@ -0,0 +1,33 @@ +# SLM-308 (LAR2-02): bounded-distance value labels vs mutation-count + +**Verdict: `rejected`** (fixture-scale matched arms; not a ship claim) + +## Preregistered thresholds (locked before results) + +- beam regret improvement (A−B) >= 0.05 +- rank correlation improvement (B−A) >= 0.1 +- rule: adopted iff both thresholds are met, else rejected + +## Headline + +| metric | arm A (mutation_count) | arm B (bounded_distance) | improvement | +| --- | --- | --- | --- | +| rank correlation | 0.4330 | 0.5052 | 0.0722 | +| concordance | 0.7500 | 0.7917 | 0.0417 | +| Brier | 0.0308 | 0.0043 | 0.0265 | +| beam regret (mean) | 0.8889 | 0.8889 | 0.0000 | + +## Coverage + oracle cost + +- UNKNOWN coverage 0.175 (7/40), measurable 17, oracle 661.7 ms/state, 64 nodes expanded + +## Calibration (arm B, distance bins d=0..8+) + +| d | n | mean predicted value | mean target | +| --- | --- | --- | --- | +| 1 | 8 | 0.787 | 0.875 | +| 2 | 9 | 0.785 | 0.750 | + +## Honesty + +Fixture-scale matched arms; UNKNOWN/unbounded states are excluded from distance-referenced metrics and counted, never coerced. A fixture verdict is wiring/label evidence, not a production ship claim. diff --git a/package-lock.json b/package-lock.json index c16585b33..31e6abda1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,10 +7,6 @@ "": { "name": "slm-training-web", "version": "0.1.0", - "dependencies": { - "langsmith": "^0.8.7", - "openai": "^6.49.0" - }, "devDependencies": { "@agentv/core": "4.42.4", "@playwright/mcp": "0.0.78", @@ -2700,12 +2696,6 @@ "node": ">= 0.6" } }, - "node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "license": "MIT" - }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -3377,39 +3367,6 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/langsmith": { - "version": "0.8.7", - "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.8.7.tgz", - "integrity": "sha512-QWcc7JwmGy+sPRJwqCf9xLiiSIhwc/i7oVwjQezkx6UzyFlokME3Wxy80qKqirTyJvWG1S9ylqtxBDpFRpas5g==", - "license": "MIT", - "dependencies": { - "p-queue": "6.6.2" - }, - "peerDependencies": { - "@opentelemetry/api": "*", - "@opentelemetry/exporter-trace-otlp-proto": "*", - "@opentelemetry/sdk-trace-base": "*", - "openai": "*", - "ws": ">=7" - }, - "peerDependenciesMeta": { - "@opentelemetry/api": { - "optional": true - }, - "@opentelemetry/exporter-trace-otlp-proto": { - "optional": true - }, - "@opentelemetry/sdk-trace-base": { - "optional": true - }, - "openai": { - "optional": true - }, - "ws": { - "optional": true - } - } - }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -3614,61 +3571,6 @@ "wrappy": "1" } }, - "node_modules/openai": { - "version": "6.49.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-6.49.0.tgz", - "integrity": "sha512-aYCc0C6L864eR6WSYIwQGyXriw/nIyZx0ObvhzOEVuk0zoBDpynjSbrionWI7q65B5H8jJX0DXR9snEzM6bfPg==", - "license": "Apache-2.0", - "peerDependencies": { - "@aws-sdk/credential-provider-node": ">=3.972.0 <4", - "@smithy/hash-node": ">=4.3.0 <5", - "@smithy/signature-v4": ">=5.4.0 <6", - "ws": "^8.18.0", - "zod": "^3.25 || ^4.0" - }, - "peerDependenciesMeta": { - "@aws-sdk/credential-provider-node": { - "optional": true - }, - "@smithy/hash-node": { - "optional": true - }, - "@smithy/signature-v4": { - "optional": true - }, - "ws": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, - "node_modules/p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/p-queue": { - "version": "6.6.2", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", - "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", - "license": "MIT", - "dependencies": { - "eventemitter3": "^4.0.4", - "p-timeout": "^3.2.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/p-retry": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", @@ -3683,18 +3585,6 @@ "node": ">=8" } }, - "node_modules/p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "license": "MIT", - "dependencies": { - "p-finally": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", diff --git a/package.json b/package.json index 1d4c0d837..05e70d356 100644 --- a/package.json +++ b/package.json @@ -27,9 +27,5 @@ }, "engines": { "node": ">=20 <23" - }, - "dependencies": { - "langsmith": "^0.8.7", - "openai": "^6.49.0" } } diff --git a/scripts/run_agentv_eval.mjs b/scripts/run_agentv_eval.mjs index 5a0d73657..7e18dd2b3 100644 --- a/scripts/run_agentv_eval.mjs +++ b/scripts/run_agentv_eval.mjs @@ -31,31 +31,33 @@ async function publishLangSmithSummary({ traceId, runId, experiment, result }) { const parentRunId = traceUuid(traceId); if (!langsmithEnabled() || !parentRunId) return; try { - const { Client } = await import("langsmith"); - const client = new Client({ - apiKey: process.env.LANGSMITH_API_KEY, - apiUrl: process.env.LANGSMITH_ENDPOINT, - workspaceId: process.env.LANGSMITH_WORKSPACE_ID, - omitTracedRuntimeInfo: true, - }); const now = new Date().toISOString(); - await client.createRun({ - id: randomUUID(), - trace_id: parentRunId, - parent_run_id: parentRunId, - project_name: process.env.LANGSMITH_PROJECT || "slm-training", - name: "agentv.publication", - run_type: "tool", - inputs: { run_id: runId, experiment }, - outputs: { summary: result.summary }, - start_time: now, - end_time: now, - extra: { metadata: { w3c_trace_id: traceId, sdk: "@agentv/core" } }, + const endpoint = `${(process.env.LANGSMITH_ENDPOINT || "https://api.smith.langchain.com").replace(/\/$/, "")}/runs`; + const response = await fetch(endpoint, { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": process.env.LANGSMITH_API_KEY, + ...(process.env.LANGSMITH_WORKSPACE_ID + ? { "x-tenant-id": process.env.LANGSMITH_WORKSPACE_ID } + : {}), + }, + body: JSON.stringify({ + id: randomUUID(), + trace_id: parentRunId, + parent_run_id: parentRunId, + project_name: process.env.LANGSMITH_PROJECT || "slm-training", + name: "agentv.publication", + run_type: "tool", + inputs: { run_id: runId, experiment }, + outputs: { summary: result.summary }, + start_time: now, + end_time: now, + extra: { metadata: { w3c_trace_id: traceId, sdk: "@agentv/core" } }, + }), + signal: AbortSignal.timeout(500), }); - await Promise.race([ - client.flush(), - new Promise((resolve) => setTimeout(resolve, 500)), - ]); + if (!response.ok) throw new Error(`HTTP ${response.status}`); } catch (error) { console.warn(`LangSmith AgentV export failed: ${String(error)}`); } diff --git a/scripts/run_slm308_distance_value.py b/scripts/run_slm308_distance_value.py new file mode 100644 index 000000000..e02c7a500 --- /dev/null +++ b/scripts/run_slm308_distance_value.py @@ -0,0 +1,627 @@ +#!/usr/bin/env python3 +"""SLM-308 (LAR2-02): matched fixture experiment — mutation-count vs bounded-distance value labels. + +Question: does replacing the X22 tree-edit model's mutation-count value +labels (``1 - applied/(max_chain+1)``) with SLM-308 oracle labels (normalized +cost-to-go from the bounded reverse-BFS distance oracle + pairwise +parent/improving-child ranking) improve value quality and value-guided beam +selection, at equal model / data / optimizer budget? + +Preregistered (written into the output payload BEFORE any result): + +- primary: beam selection regret vs oracle-best must improve by **>= 0.05** + absolute (arm A regret − arm B regret, distance units); +- secondary: Spearman rank correlation between value and oracle distance + must improve by **>= 0.10** (arm B − arm A); +- verdict ``adopted`` iff both thresholds are met, else ``rejected`` — + honestly computed from the measured arms, never narrated. + +Arms share the tiny fixture corpus, model init seed, mutation chains, batch +order, and optimizer budget; they differ ONLY in ``value_label_mode``. +Evaluation covers near-gold states (gold mutated 1..3 edits) AND +seed-trajectory states (random walks from the decode seed): Spearman rank +correlation, pairwise concordance, Brier + per-bin calibration over distance +bins d=0..8+, beam selection regret vs oracle-best, UNKNOWN coverage, and +oracle cost (nodes expanded / ms). States the oracle cannot measure (UNKNOWN +or unbounded) are excluded from distance-referenced metrics and counted, +never coerced. + +Writes ``docs/design/iter-slm308-distance-value-20260724.{json,md}``. + +Example: + python -m scripts.run_slm308_distance_value --steps 8 +""" + +from __future__ import annotations + +import argparse +import json +import random +import time +from pathlib import Path + +import torch + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm308_distance_oracle import ( + DistanceKind, + cache_stats, + clear_caches, + distance_to_target, + effective_distance, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_STOP, + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + TreeEditSpace, + parse_statements, + render_statements, +) +from slm_training.versioning import build_version_stamp + +EXPERIMENT_ID = "slm308-distance-value" +DEFAULT_JSON_OUT = Path("docs/design/iter-slm308-distance-value-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm308-distance-value-20260724.md") + +# PREREGISTERED improvement thresholds (locked before any run; do not edit +# after outcomes are visible — deviations are append-only and exploratory). +PREREGISTERED_THRESHOLDS = { + "beam_regret_improvement_min": 0.05, + "rank_correlation_improvement_min": 0.10, + "direction": "arm_b(bounded_distance) minus arm_a(mutation_count)", + "verdict_rule": "adopted iff both thresholds are met, else rejected", +} + +EVAL_MAX_DEPTH = 8 +EVAL_NODE_BUDGET = 8 + +# Tiny fixture corpus: small valid programs with 1-2 placeholders. +FIXTURE_PROGRAMS: list[tuple[str, str, list[str]]] = [ + ("hero with cta", 'root = Stack([t, b], "column")\nt = TextContent(":hero.title")\nb = Button(":cta.label")', [":hero.title", ":cta.label"]), + ("simple text", 'root = Stack([t], "column")\nt = TextContent(":body")', [":body"]), + ("form with input", 'root = Form([i, s], "column")\ni = TextInput(":form.email")\ns = Button(":form.submit")', [":form.email", ":form.submit"]), + ("card with image", 'root = Stack([c], "column")\nc = Card([im, t], "column")\nim = Image(":img.src")\nt = TextContent(":img.caption")', [":img.src", ":img.caption"]), + ("two texts", 'root = Stack([a, b], "column")\na = TextContent(":title")\nb = TextContent(":subtitle")', [":title", ":subtitle"]), + ("button only", 'root = Stack([b], "column")\nb = Button(":cta")', [":cta"]), + ("nested card", 'root = Stack([c], "column")\nc = Card([t], "column")\nt = TextContent(":card.body")', [":card.body"]), + ("form pair", 'root = Form([i], "column")\ni = TextInput(":q")', [":q"]), +] + +SEED_SOURCE = 'root = Stack([], "column")' + + +def build_fixture_records() -> list[ExampleRecord]: + return [ + ExampleRecord( + id=f"fixture_{i:02d}", + prompt=prompt, + openui=openui, + placeholders=list(placeholders), + split="train", + ) + for i, (prompt, openui, placeholders) in enumerate(FIXTURE_PROGRAMS) + ] + + +# --- metrics ----------------------------------------------------------------- + + +def spearman(xs: list[float], ys: list[float]) -> float | None: + """Spearman rank correlation; None when undefined (ties everywhere).""" + + def ranks(values: list[float]) -> list[float]: + order = sorted(range(len(values)), key=lambda i: values[i]) + out = [0.0] * len(values) + i = 0 + while i < len(order): + j = i + while j + 1 < len(order) and values[order[j + 1]] == values[order[i]]: + j += 1 + mean_rank = (i + j) / 2.0 + 1.0 + for k in range(i, j + 1): + out[order[k]] = mean_rank + i = j + 1 + return out + + if len(xs) < 3: + return None + rx, ry = ranks(xs), ranks(ys) + n = len(rx) + mx, my = sum(rx) / n, sum(ry) / n + cov = sum((a - mx) * (b - my) for a, b in zip(rx, ry)) + vx = sum((a - mx) ** 2 for a in rx) + vy = sum((b - my) ** 2 for b in ry) + if vx == 0 or vy == 0: + return None + return cov / (vx * vy) ** 0.5 + + +def concordance(preds: list[float], refs: list[float]) -> float | None: + """Fraction of comparable pairs the prediction orders correctly (ties in + reference skipped; prediction ties count as half).""" + total = 0 + correct = 0.0 + for i in range(len(preds)): + for j in range(i + 1, len(preds)): + if refs[i] == refs[j]: + continue + total += 1 + want = refs[i] < refs[j] + if preds[i] == preds[j]: + correct += 0.5 + elif (preds[i] < preds[j]) == want: + correct += 1.0 + return (correct / total) if total else None + + +def calibration_bins( + preds: list[float], refs: list[float], max_depth: int = EVAL_MAX_DEPTH +) -> list[dict[str, float | int]]: + """Per distance bin d=0..8+: mean predicted value vs mean normalized + cost-to-go target (1 - d/max_depth). Bins are by rounded reference + distance; d>=8 folds into the final bin.""" + bins: dict[int, list[tuple[float, float]]] = {} + for pred, ref in zip(preds, refs): + d = min(int(round(ref)), max_depth) + bins.setdefault(d, []).append((pred, 1.0 - min(ref, max_depth) / max_depth)) + return [ + { + "distance_bin": d, + "n": len(rows), + "mean_predicted_value": sum(r[0] for r in rows) / len(rows), + "mean_target": sum(r[1] for r in rows) / len(rows), + } + for d, rows in sorted(bins.items()) + ] + + +# --- eval states --------------------------------------------------------------- + + +def build_eval_states( + records: list[ExampleRecord], space: TreeEditSpace, *, seed: int = 20260724 +) -> list[dict]: + """Deterministic near-gold and seed-trajectory states per record.""" + states: list[dict] = [] + for idx, record in enumerate(records): + gold = parse_statements(record.openui) + assert gold is not None + inventory = [ + p if p.startswith(":") else f":{p}" for p in record.placeholders + ] + # Near-gold: mutate gold 1..3 edits. + rng = random.Random(seed + idx) + for k in (1, 2, 3): + current = gold + applied = 0 + for _ in range(k): + step = space.sample_mutation(current, inventory, rng) + if step is None: + break + current, _inverse = step + applied += 1 + if applied == 0: + continue + states.append( + { + "record_idx": idx, + "origin": "near_gold", + "witness": applied, + "statements": current, + "target": gold, + "inventory": inventory, + } + ) + # Seed-trajectory: random walks from the decode seed (1, 3 edits). + seed_statements = parse_statements(SEED_SOURCE) + assert seed_statements is not None + for k in (1, 3): + current = seed_statements + applied = 0 + for _ in range(k): + step = space.sample_mutation(current, inventory, rng) + if step is None: + break + current, _inverse = step + applied += 1 + if applied == 0: + continue + states.append( + { + "record_idx": idx, + "origin": "seed_trajectory", + "witness": None, # not a gold mutation chain: no witness + "statements": current, + "target": gold, + "inventory": inventory, + } + ) + return states + + +def label_eval_states(states: list[dict], space: TreeEditSpace) -> dict: + """Oracle labels for every eval state (shared across arms) + oracle cost.""" + clear_caches() + start = time.perf_counter() + n_unknown = 0 + for entry in states: + label = distance_to_target( + entry["statements"], + entry["target"], + space=space, + inventory=entry["inventory"], + max_depth=EVAL_MAX_DEPTH, + node_budget=EVAL_NODE_BUDGET, + upper_bound_witness=entry["witness"], + ) + entry["label"] = label + entry["ref_distance"] = effective_distance(label) + if label.kind is DistanceKind.UNKNOWN: + n_unknown += 1 + elapsed_ms = (time.perf_counter() - start) * 1000.0 + stats = cache_stats() + return { + "n_states": len(states), + "n_unknown": n_unknown, + "unknown_coverage": n_unknown / max(len(states), 1), + "n_measurable": sum(1 for e in states if e["ref_distance"] is not None), + "wall_ms_total": elapsed_ms, + "ms_per_state": elapsed_ms / max(len(states), 1), + "cache": stats, + } + + +# --- training ------------------------------------------------------------------ + + +def train_arm( + records: list[ExampleRecord], + mode: str, + *, + steps: int, + batch_size: int, + seed: int, +) -> TreeEditDiffusionModel: + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + seed=seed, + value_label_mode=mode, + context_backend="scratch", + max_chain=3, + ) + model = TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=3e-3) + order = random.Random(0) # identical batch order across arms + model.train() + for _ in range(steps): + batch = [records[order.randrange(len(records))] for _ in range(batch_size)] + loss = model.training_loss(batch) + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +# --- arm scoring ----------------------------------------------------------------- + + +@torch.no_grad() +def _model_value(model: TreeEditDiffusionModel, prompt: str, source: str) -> float: + ctx, ctx_pad = model._encode_context([prompt]) + out = model.policy( + model._state_batch([source]), model.tokenizer.pad_id, ctx, ctx_pad + ) + return float(out["value"][0]) + + +@torch.no_grad() +def beam_regret( + model: TreeEditDiffusionModel, + record: ExampleRecord, + entry: dict, +) -> float | None: + """Value-guided single-step selection regret vs oracle-best, in distance + units. Candidates mirror decode: STOP plus the policy's top applicable + edits; the value head picks; the oracle scores every candidate. None when + any candidate is oracle-unmeasurable (excluded, never coerced).""" + inventory = entry["inventory"] + statements = entry["statements"] + prompt = model._format_context( + record.prompt, design_md=record.design_md, slot_contract=inventory + ) + ctx, ctx_pad = model._encode_context([prompt]) + out = model.policy( + model._state_batch([render_statements(statements)]), + model.tokenizer.pad_id, + ctx, + ctx_pad, + ) + candidates = model._enumerate_edits(out, 0, len(statements), len(inventory)) + # (child statements or None for STOP) + options: list[list | None] = [] + expanded = 0 + for _score, edit in candidates: + if expanded >= model.config.expand_per_state: + break + if edit.action == ACTION_STOP: + options.append(None) + expanded += 1 + continue + child = model.space.apply(statements, edit, inventory) + if child is None: + continue + options.append(child) + expanded += 1 + if not options: + return None + child_sources = [ + render_statements(child) if child is not None else render_statements(statements) + for child in options + ] + values = model.policy( + model._state_batch(child_sources), + model.tokenizer.pad_id, + ctx.expand(len(options), -1, -1), + ctx_pad.expand(len(options), -1), + )["value"] + distances: list[float] = [] + for child in options: + # A candidate child of a witness-backed state is one edit further from + # gold along the same witness path: hi = witness + 1 is proven. + witness = ( + entry["witness"] + 1 if entry["witness"] is not None else None + ) + label = distance_to_target( + child if child is not None else statements, + entry["target"], + space=model.space, + inventory=inventory, + max_depth=EVAL_MAX_DEPTH, + node_budget=EVAL_NODE_BUDGET, + upper_bound_witness=witness, + ) + ref = effective_distance(label) + if ref is None: + return None + distances.append(ref) + pick = max(range(len(options)), key=lambda i: float(values[i])) + return distances[pick] - min(distances) + + +def score_arm( + model: TreeEditDiffusionModel, + records: list[ExampleRecord], + states: list[dict], +) -> dict: + measurable = [e for e in states if e["ref_distance"] is not None] + preds: list[float] = [] + refs: list[float] = [] + regrets: list[float] = [] + n_regret_excluded = 0 + for entry in measurable: + record = records[entry["record_idx"]] + prompt = model._format_context( + record.prompt, + design_md=record.design_md, + slot_contract=entry["inventory"], + ) + value = _model_value(model, prompt, render_statements(entry["statements"])) + preds.append(value) + refs.append(entry["ref_distance"]) + regret = beam_regret(model, record, entry) + if regret is None: + n_regret_excluded += 1 + else: + regrets.append(regret) + # Higher value must mean closer => correlation against NEGATIVE distance. + rank = spearman(preds, [-r for r in refs]) + conc = concordance(preds, [-r for r in refs]) + targets = [1.0 - min(r, EVAL_MAX_DEPTH) / EVAL_MAX_DEPTH for r in refs] + brier = ( + sum((p - t) ** 2 for p, t in zip(preds, targets)) / len(preds) + if preds + else None + ) + by_origin: dict[str, dict] = {} + for origin in ("near_gold", "seed_trajectory"): + sel = [ + (p, r) + for p, r, e in zip(preds, refs, measurable) + if e["origin"] == origin + ] + if sel: + by_origin[origin] = { + "n": len(sel), + "rank_correlation": spearman( + [p for p, _ in sel], [-r for _, r in sel] + ), + } + return { + "n_measurable": len(measurable), + "rank_correlation": rank, + "concordance": conc, + "brier": brier, + "calibration_bins": calibration_bins(preds, refs), + "beam_regret_mean": (sum(regrets) / len(regrets)) if regrets else None, + "beam_regret_n": len(regrets), + "beam_regret_excluded": n_regret_excluded, + "by_origin": by_origin, + } + + +# --- report --------------------------------------------------------------------- + + +def build_report( + records: list[ExampleRecord], + *, + steps: int, + batch_size: int, + seed: int, +) -> dict: + space = TreeEditSpace() + states = build_eval_states(records, space) + # Oracle labels are arm-independent: label once, share across arms (the + # oracle cache is keyed by grammar/action versions + AST hashes). + oracle = label_eval_states(states, space) + arms: dict[str, dict] = {} + for arm, mode in (("arm_a", "mutation_count"), ("arm_b", "bounded_distance")): + model = train_arm(records, mode, steps=steps, batch_size=batch_size, seed=seed) + metrics = score_arm(model, records, states) + arms[arm] = { + "value_label_mode": mode, + "metrics": metrics, + "training_metrics": model.last_training_metrics, + } + a, b = arms["arm_a"]["metrics"], arms["arm_b"]["metrics"] + + def _delta(key: str, reverse: bool = False) -> float | None: + if a.get(key) is None or b.get(key) is None: + return None + return (a[key] - b[key]) if reverse else (b[key] - a[key]) + + improvements = { + "rank_correlation": _delta("rank_correlation"), + "concordance": _delta("concordance"), + "beam_regret": _delta("beam_regret_mean", reverse=True), + "brier": _delta("brier", reverse=True), + } + rank_ok = ( + improvements["rank_correlation"] is not None + and improvements["rank_correlation"] + >= PREREGISTERED_THRESHOLDS["rank_correlation_improvement_min"] + ) + regret_ok = ( + improvements["beam_regret"] is not None + and improvements["beam_regret"] + >= PREREGISTERED_THRESHOLDS["beam_regret_improvement_min"] + ) + verdict = "adopted" if (rank_ok and regret_ok) else "rejected" + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-308", + "question": ( + "Do bounded-distance oracle value labels beat mutation-count value " + "labels at equal fixture budget (rank correlation + beam regret)?" + ), + "preregistered_thresholds": PREREGISTERED_THRESHOLDS, + "config": { + "steps": steps, + "batch_size": batch_size, + "seed": seed, + "n_records": len(records), + "eval_max_depth": EVAL_MAX_DEPTH, + "eval_node_budget": EVAL_NODE_BUDGET, + }, + "oracle_cost": oracle, + "arms": arms, + "improvements": improvements, + "threshold_checks": { + "rank_correlation_ok": rank_ok, + "beam_regret_ok": regret_ok, + }, + "verdict": verdict, + "honesty": ( + "Fixture-scale matched arms; UNKNOWN/unbounded states are excluded " + "from distance-referenced metrics and counted, never coerced. A " + "fixture verdict is wiring/label evidence, not a production ship " + "claim." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm308_distance_value", + "harness.experiments.slm299_edit_reachability", + ) + return payload + + +def render_markdown(payload: dict) -> str: + a = payload["arms"]["arm_a"]["metrics"] + b = payload["arms"]["arm_b"]["metrics"] + imp = payload["improvements"] + thr = payload["preregistered_thresholds"] + + def fmt(x: object) -> str: + return "n/a" if x is None else (f"{x:.4f}" if isinstance(x, float) else str(x)) + + lines = [ + "# SLM-308 (LAR2-02): bounded-distance value labels vs mutation-count", + "", + f"**Verdict: `{payload['verdict']}`** (fixture-scale matched arms; not a ship claim)", + "", + "## Preregistered thresholds (locked before results)", + "", + f"- beam regret improvement (A−B) >= {thr['beam_regret_improvement_min']}", + f"- rank correlation improvement (B−A) >= {thr['rank_correlation_improvement_min']}", + f"- rule: {thr['verdict_rule']}", + "", + "## Headline", + "", + "| metric | arm A (mutation_count) | arm B (bounded_distance) | improvement |", + "| --- | --- | --- | --- |", + f"| rank correlation | {fmt(a['rank_correlation'])} | {fmt(b['rank_correlation'])} | {fmt(imp['rank_correlation'])} |", + f"| concordance | {fmt(a['concordance'])} | {fmt(b['concordance'])} | {fmt(imp['concordance'])} |", + f"| Brier | {fmt(a['brier'])} | {fmt(b['brier'])} | {fmt(imp['brier'])} |", + f"| beam regret (mean) | {fmt(a['beam_regret_mean'])} | {fmt(b['beam_regret_mean'])} | {fmt(imp['beam_regret'])} |", + "", + "## Coverage + oracle cost", + "", + ] + oracle = payload["oracle_cost"] + lines.append( + f"- UNKNOWN coverage {oracle['unknown_coverage']:.3f} " + f"({oracle['n_unknown']}/{oracle['n_states']}), measurable " + f"{oracle['n_measurable']}, oracle {oracle['ms_per_state']:.1f} ms/state, " + f"{oracle['cache']['map_nodes']} nodes expanded" + ) + lines += [ + "", + "## Calibration (arm B, distance bins d=0..8+)", + "", + "| d | n | mean predicted value | mean target |", + "| --- | --- | --- | --- |", + ] + for row in b["calibration_bins"]: + lines.append( + f"| {row['distance_bin']} | {row['n']} | " + f"{row['mean_predicted_value']:.3f} | {row['mean_target']:.3f} |" + ) + lines += [ + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=8) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + args = parser.parse_args(argv) + + records = build_fixture_records() + payload = build_report( + records, steps=args.steps, batch_size=args.batch_size, seed=args.seed + ) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"verdict={payload['verdict']} " + f"rank_imp={payload['improvements']['rank_correlation']} " + f"regret_imp={payload['improvements']['beam_regret']} " + f"-> {args.json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm308_distance_oracle.py b/src/slm_training/harnesses/experiments/slm308_distance_oracle.py new file mode 100644 index 000000000..dacfe5c7d --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm308_distance_oracle.py @@ -0,0 +1,398 @@ +"""SLM-308 (LAR2-02): bounded distance oracle over canonical AST fingerprints. + +Replaces mutation-count value labels with an exact/bounded/UNKNOWN cost-to-go +signal for the X22 tree-edit model. The oracle runs a **reverse BFS from the +target state** over alpha-invariant canonical fingerprints (``_canonical_key`` +from SLM-299) using the extended 11-action space's transitions. Because every +SLM-305 action is invertible inside the space, the transition graph is +undirected: enumerating forward children of the target is exactly enumerating +its reverse neighborhood, so BFS depth from the target is the true shortest +edit distance for every state it reaches. + +Three labels, and only three: + +- ``EXACT(d)`` — proven shortest distance ``d``: the BFS reached the state and + BFS layering certifies no shorter path exists. +- ``BOUNDED(lo, hi)`` — proof of bounds only, with a machine-readable reason. + ``lo`` comes from fully explored BFS layers (the state was not within the + deepest fully explored depth); ``hi`` comes from a witness path length + (e.g. the mutation chain that produced a training state) when one is + supplied. An invariant proof yields ``BOUNDED(None, None)`` — the target is + unreachable under the exact action set, which is a *proof*, never a budget + artifact. +- ``UNKNOWN`` — node budget exhausted with a live frontier. UNKNOWN is + explicit and is never conflated with far or unreachable; downstream + consumers (value loss) must exclude it, never coerce it. + +Results are cached keyed by ``(action_schema_version, grammar_version, +inventory_hash, target_ast_hash, state_ast_hash, max_depth, witness)``. The +action schema version is the SLM-299 component version (it watches the edit +space); there is no explicit grammar backend version, so the grammar version +is a sha256 fragment of ``dsl/grammars/openui.lark`` — a grammar or action +change is a guaranteed cache miss. + +The oracle is **training-time only**: it reads the gold target and must never +be reachable from any inference/decode path. +""" + +from __future__ import annotations + +import hashlib +import time +from collections import deque +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Sequence + +from slm_training.harness_core.versioning import component_version +from slm_training.harnesses.experiments.slm299_edit_reachability import ( + _canonical_key, + _check_invariants, + _enumerate_children, + _normalize_inventory, +) +from slm_training.models.tree_edit_diffusion import Statement, TreeEditSpace + +__all__ = [ + "DistanceKind", + "DistanceLabel", + "action_schema_version", + "cache_stats", + "clear_caches", + "distance_to_target", + "effective_distance", + "grammar_version", +] + +EXPERIMENT_ID = "slm308-distance-value" + +_GRAMMAR_PATH = ( + Path(__file__).resolve().parents[2] / "dsl" / "grammars" / "openui.lark" +) + +# Reason codes. +REASON_REACHED = "reached" +REASON_DEPTH_BOUND = "depth_bound" # fully explored to max_depth; not found +REASON_FRONTIER_EXHAUSTED = "frontier_exhausted" # frontier died inside bound +REASON_BUDGET = "budget" # node budget exhausted with a live frontier +REASON_INVARIANT_PREFIX = "invariant:" # + slm299 invariant reason code + + +class DistanceKind(str, Enum): + """Proof status of one state→target distance query.""" + + EXACT = "EXACT" + BOUNDED = "BOUNDED" + UNKNOWN = "UNKNOWN" + + +@dataclass +class DistanceLabel: + """Distance proof for one (state, target) pair. + + ``distance`` is set only for EXACT. ``lo``/``hi`` are set for BOUNDED when + the corresponding bound is proven (``hi=None`` means no finite upper bound + was proven — e.g. an invariant-fired unreachable pair). ``nodes_expanded`` + is the oracle's own search cost for the BFS that decided this label. + """ + + kind: DistanceKind + reason: str + distance: int | None = None + lo: int | None = None + hi: int | None = None + nodes_expanded: int = 0 + cache_hit: bool = False + + def value_target(self, max_depth: int) -> float | None: + """Normalized cost-to-go value target in [0, 1] (higher = closer). + + EXACT ``d`` maps to ``1 - d/max_depth``; BOUNDED with both finite + bounds maps to the flagged midpoint; BOUNDED without finite bounds and + UNKNOWN return None — the state is excluded from value loss, never + coerced. + """ + eff = effective_distance(self) + if eff is None: + return None + return max(0.0, 1.0 - min(eff, float(max_depth)) / float(max_depth)) + + def to_dict(self) -> dict[str, object]: + return { + "kind": self.kind.value, + "reason": self.reason, + "distance": self.distance, + "lo": self.lo, + "hi": self.hi, + "nodes_expanded": self.nodes_expanded, + "cache_hit": self.cache_hit, + } + + +def effective_distance(label: DistanceLabel) -> float | None: + """Comparable distance estimate: exact ``d`` or the flagged bound midpoint. + + None when the label carries no usable finite estimate (UNKNOWN, or + invariant-proven unreachable with no witness). Callers must treat None as + "exclude", never as far/unreachable. + """ + if label.kind is DistanceKind.EXACT: + return float(label.distance) + if label.kind is DistanceKind.BOUNDED and label.lo is not None and label.hi is not None: + return (label.lo + label.hi) / 2.0 + return None + + +def action_schema_version() -> str: + """Version of the action space the oracle enumerates (SLM-299 component).""" + return component_version("harness.experiments.slm299_edit_reachability") + + +def grammar_version() -> str: + """Grammar identity: sha256 fragment of the Lark grammar the parser uses.""" + try: + digest = hashlib.sha256(_GRAMMAR_PATH.read_bytes()).hexdigest() + except OSError: + return "unknown" + return f"sha256:{digest[:16]}" + + +def _ast_hash(statements: list[Statement]) -> str: + return hashlib.sha256(_canonical_key(statements).encode("utf-8")).hexdigest()[:16] + + +def _inventory_hash(inventory: Sequence[str]) -> str: + return hashlib.sha256("\n".join(inventory).encode("utf-8")).hexdigest()[:12] + + +# --- caches ----------------------------------------------------------------- + + +@dataclass +class _TargetMap: + """BFS distance map from one target, plus what the search proved.""" + + dist: dict[str, int] = field(default_factory=dict) + deepest_full_depth: int = -1 # every state at this depth was explored + budget_exhausted: bool = False + frontier_died: bool = False # frontier provably emptied before max_depth + nodes_expanded: int = 0 + + +# target map: (action_ver, grammar_ver, inventory_hash, target_ast_hash, max_depth) +_TARGET_MAPS: dict[tuple[str, ...], _TargetMap] = {} +# label cache: target-map key + (state_ast_hash, witness) +_LABELS: dict[tuple, DistanceLabel] = {} +_STATS = {"label_hits": 0, "label_misses": 0, "map_builds": 0} + + +def cache_stats() -> dict[str, int | float]: + return { + **_STATS, + "target_maps": len(_TARGET_MAPS), + "labels": len(_LABELS), + "map_nodes": sum(m.nodes_expanded for m in _TARGET_MAPS.values()), + } + + +def clear_caches() -> None: + _TARGET_MAPS.clear() + _LABELS.clear() + for key in _STATS: + _STATS[key] = 0 + + +def _build_target_map( + target: list[Statement], + inventory: list[str], + space: TreeEditSpace, + *, + max_depth: int, + node_budget: int, +) -> _TargetMap: + """Layered BFS from the target over canonical fingerprints. + + Transitions are the real extended-space children via SLM-299's + ``_enumerate_children`` applied through ``TreeEditSpace.apply`` — the + analyzed space is the deployed space by construction. Because every action + is invertible in-space, BFS depth from the target is the exact shortest + distance for every reached state. + """ + result = _TargetMap() + target_key = _canonical_key(target) + result.dist[target_key] = 0 + frontier: deque[list[Statement]] = deque([target]) + depth = 0 + while frontier and depth < max_depth: + # Expand one full BFS layer; only fully explored layers raise + # ``deepest_full_depth`` (a partial layer proves no lower bound). + layer_size = len(frontier) + layer_complete = True + for _ in range(layer_size): + if result.nodes_expanded >= node_budget: + result.budget_exhausted = True + layer_complete = False + break + statements = frontier.popleft() + result.nodes_expanded += 1 + # Visited pruning is a pure search-efficiency hook: states already + # keyed are rejected pre-validation inside ``apply``; it never + # changes which distinct states are reachable. + visited = set(result.dist) + for child, _action in _enumerate_children( + space, statements, inventory, mode="extended", visited=visited + ): + key = _canonical_key(child) + if key in result.dist: + continue + result.dist[key] = depth + 1 + frontier.append(child) + if not layer_complete: + break + result.deepest_full_depth = depth + depth += 1 + if not frontier and not result.budget_exhausted: + # Frontier provably died: every reachable state within the + # MAX_STMTS-bounded space was enumerated, so unreached states are + # beyond any depth the space connects — still reported via bounds. + result.frontier_died = True + return result + + +def distance_to_target( + statements: list[Statement], + target_statements: list[Statement], + *, + space: TreeEditSpace, + inventory: Sequence[str] = (), + max_depth: int = 8, + node_budget: int = 600, + upper_bound_witness: int | None = None, +) -> DistanceLabel: + """Distance proof from ``statements`` to ``target_statements``. + + ``upper_bound_witness`` is a proven path length (e.g. the mutation chain + that produced a training state from the target); it only ever tightens + ``hi``, never ``lo``. UNKNOWN is returned on budget exhaustion and is + never conflated with far or unreachable. + """ + versions = (action_schema_version(), grammar_version()) + inv = _normalize_inventory(inventory) + inv_hash = _inventory_hash(inv) + state_hash = _ast_hash(statements) + target_hash = _ast_hash(target_statements) + map_key = (*versions, inv_hash, target_hash, str(max_depth)) + label_key = (map_key, state_hash, upper_bound_witness, node_budget) + + cached = _LABELS.get(label_key) + if cached is not None: + _STATS["label_hits"] += 1 + return DistanceLabel( + kind=cached.kind, + reason=cached.reason, + distance=cached.distance, + lo=cached.lo, + hi=cached.hi, + nodes_expanded=cached.nodes_expanded, + cache_hit=True, + ) + _STATS["label_misses"] += 1 + + if state_hash == target_hash: + label = DistanceLabel( + kind=DistanceKind.EXACT, reason=REASON_REACHED, distance=0 + ) + _LABELS[label_key] = label + return label + + # Structural impossibility proofs over the EXACT extended action set + # (container_add is a real capability of the extended space). + fired = _check_invariants( + target_statements, + statements, + inv, + space, + frozenset({"container_add"}), + extended=True, + ) + if fired is not None: + label = DistanceLabel( + kind=DistanceKind.BOUNDED, + reason=f"{REASON_INVARIANT_PREFIX}{fired}", + lo=None, + hi=None, + ) + _LABELS[label_key] = label + return label + + target_map = _TARGET_MAPS.get(map_key) + if target_map is None: + _STATS["map_builds"] += 1 + target_map = _build_target_map( + target_statements, + inv, + space, + max_depth=max_depth, + node_budget=node_budget, + ) + _TARGET_MAPS[map_key] = target_map + + state_key = _canonical_key(statements) + exact = target_map.dist.get(state_key) + if exact is not None: + label = DistanceLabel( + kind=DistanceKind.EXACT, + reason=REASON_REACHED, + distance=exact, + nodes_expanded=target_map.nodes_expanded, + ) + elif target_map.budget_exhausted: + # Live frontier when the budget cut the search: nothing is proven + # about this state beyond the fully explored layers, and a witness + # alone never upgrades an UNKNOWN to bounded-exact. + if upper_bound_witness is not None and target_map.deepest_full_depth >= 0: + label = DistanceLabel( + kind=DistanceKind.BOUNDED, + reason=REASON_BUDGET, + lo=target_map.deepest_full_depth + 2, + hi=upper_bound_witness, + nodes_expanded=target_map.nodes_expanded, + ) + else: + label = DistanceLabel( + kind=DistanceKind.UNKNOWN, + reason=REASON_BUDGET, + nodes_expanded=target_map.nodes_expanded, + ) + else: + if target_map.frontier_died: + # Complete proof: the target's connected component was fully + # enumerated and the state is not in it — unreachable, with the + # same status as an invariant proof (no finite bounds). + label = DistanceLabel( + kind=DistanceKind.BOUNDED, + reason=REASON_FRONTIER_EXHAUSTED, + lo=None, + hi=None, + nodes_expanded=target_map.nodes_expanded, + ) + else: + # Depth bound cut a live frontier: distances up to max_depth were + # enumerated, so this state's distance is provably > max_depth. + label = DistanceLabel( + kind=DistanceKind.BOUNDED, + reason=REASON_DEPTH_BOUND, + lo=max_depth + 1, + hi=upper_bound_witness, + nodes_expanded=target_map.nodes_expanded, + ) + _LABELS[label_key] = label + return label + + +def timed_distance_to_target(**kwargs) -> tuple[DistanceLabel, float]: + """``distance_to_target`` plus wall-clock milliseconds (oracle cost evidence).""" + start = time.perf_counter() + label = distance_to_target(**kwargs) + return label, (time.perf_counter() - start) * 1000.0 diff --git a/src/slm_training/models/tree_edit_diffusion.py b/src/slm_training/models/tree_edit_diffusion.py index 13ffc430d..9d973f5b1 100644 --- a/src/slm_training/models/tree_edit_diffusion.py +++ b/src/slm_training/models/tree_edit_diffusion.py @@ -876,6 +876,41 @@ class TreeEditDiffusionConfig: schema_in_context: bool = False slot_contract_in_context: bool = True seed: int = 0 + # SLM-308 (LAR2-02): value supervision mode. "bounded_distance" (default + # for NEW trainings) labels values with the SLM-308 distance oracle's + # normalized cost-to-go plus pairwise parent/improving-child ranking; + # "mutation_count" keeps the historical 1 - applied/(max_chain+1) labels. + # Checkpoints written before this field existed load as "mutation_count" + # (see ``from_checkpoint``) for behavior parity. + value_label_mode: str = "bounded_distance" + pairwise_progress_margin: float = 0.1 + + +# SLM-308: oracle depth/budget used for training-time value labels. The +# budget is shallow by design (extended-space enumeration is parser-backed +# and expensive); states beyond the explored layers get witness-bounded +# BOUNDED labels or are excluded (UNKNOWN), never coerced. +VALUE_ORACLE_MAX_DEPTH = 8 +VALUE_ORACLE_NODE_BUDGET = 8 + + +def pairwise_progress_loss( + parent_values: torch.Tensor, + child_values: torch.Tensor, + *, + margin: float = 0.1, +) -> torch.Tensor: + """Margin ranking: an improving child (oracle-proven closer to the target) + must score at least ``margin`` higher on value than its parent. + + Higher value = closer to target. Pairs are pre-filtered by the caller to + strictly-improving, oracle-comparable pairs only — ties and unmeasurable + (UNKNOWN / unbounded) pairs never enter here, so several comparably-close + states are never forced into a strict order. + """ + if parent_values.numel() == 0: + return parent_values.new_zeros(()) + return F.relu(parent_values - child_values + margin).mean() class TreeEditPolicy(nn.Module): @@ -1029,15 +1064,45 @@ def _state_batch(self, sources: list[str]) -> torch.Tensor: # --- training --------------------------------------------------------- + def _distance_label(self, statements, target, inventory, witness: int): + """Training-time ONLY gold-distance label (SLM-308 oracle). + + Imported lazily: the oracle reads the gold target and must never be + reachable from any decode path (see the SLM-308 no-gold-at-inference + audit test). + """ + from slm_training.harnesses.experiments.slm308_distance_oracle import ( + distance_to_target, + ) + + return distance_to_target( + statements, + target, + space=self.space, + inventory=inventory, + max_depth=VALUE_ORACLE_MAX_DEPTH, + node_budget=VALUE_ORACLE_NODE_BUDGET, + upper_bound_witness=witness, + ) + def forward(self, batch: list[ExampleRecord]) -> float: return float(self.training_loss(batch).detach().cpu()) def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: + if self.config.value_label_mode not in {"mutation_count", "bounded_distance"}: + raise ValueError( + f"unknown value_label_mode {self.config.value_label_mode!r}" + ) + bounded_mode = self.config.value_label_mode == "bounded_distance" prompts: list[str] = [] states: list[str] = [] targets: list[Edit] = [] values: list[float] = [] + value_mask: list[bool] = [] + pair_rows: list[tuple[int, str]] = [] # (parent row, child source) skipped = 0 + n_bounded = 0 + n_unknown_excluded = 0 for record in batch: source = (record.openui or "").strip() statements = parse_statements(source) if source else None @@ -1054,20 +1119,24 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: slot_contract=inventory, ) if self._rng.random() < 0.2: - # Clean state: the correct move is STOP with full value. + # Clean state: the correct move is STOP with full value + # (oracle distance 0 in bounded mode, so the modes agree). prompts.append(prompt) states.append(source) targets.append(Edit(ACTION_STOP)) values.append(1.0) + value_mask.append(True) continue k = self._rng.randint(1, self.config.max_chain) current = statements + prev = None inverse: Edit | None = None applied = 0 for _ in range(k): step = self.space.sample_mutation(current, inventory, self._rng) if step is None: break + prev = current current, inverse = step applied += 1 if inverse is None: @@ -1076,16 +1145,61 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: prompts.append(prompt) states.append(render_statements(current)) targets.append(inverse) - values.append(1.0 - applied / float(self.config.max_chain + 1)) + if not bounded_mode: + values.append(1.0 - applied / float(self.config.max_chain + 1)) + value_mask.append(True) + continue + # SLM-308: normalized cost-to-go from the bounded distance oracle. + # UNKNOWN / unbounded states are excluded from value loss, never + # coerced; the mutation chain length is a proven witness upper + # bound (the inverse edits walk back to gold). + label = self._distance_label(current, statements, inventory, applied) + target_value = label.value_target(VALUE_ORACLE_MAX_DEPTH) + if target_value is None: + values.append(0.0) + value_mask.append(False) + n_unknown_excluded += 1 + else: + values.append(target_value) + value_mask.append(True) + if label.kind.value == "BOUNDED": + n_bounded += 1 + if prev is not None: + # Pairwise progress: prev is one inverse-edit closer to gold. + # The pair enters the ranking loss only when the oracle proves + # a strict improvement on comparable (finite) estimates. + from slm_training.harnesses.experiments.slm308_distance_oracle import ( # noqa: E501 + effective_distance, + ) + + child_label = self._distance_label( + prev, statements, inventory, applied - 1 + ) + d_parent = effective_distance(label) + d_child = effective_distance(child_label) + if ( + d_parent is not None + and d_child is not None + and d_child < d_parent + ): + pair_rows.append((len(states) - 1, render_statements(prev))) if not states: return torch.zeros((), device=self.device_name, requires_grad=True) + n_main = len(states) + pair_child_rows = list(range(n_main, n_main + len(pair_rows))) + states.extend(child_source for _, child_source in pair_rows) ctx, ctx_pad = self._encode_context(prompts) + if pair_rows: + # Pairwise-ranking child rows reuse their parent's prompt context. + ctx_rows = [row for row, _ in pair_rows] + ctx = torch.cat([ctx, ctx[ctx_rows]], dim=0) + ctx_pad = torch.cat([ctx_pad, ctx_pad[ctx_rows]], dim=0) out = self.policy( self._state_batch(states), self.tokenizer.pad_id, ctx, ctx_pad ) device = self.device_name action_t = torch.tensor([e.action for e in targets], device=device) - loss = F.cross_entropy(out["action"], action_t) + loss = F.cross_entropy(out["action"][:n_main], action_t) losses = {"action": float(loss.detach().cpu())} stmt_rows = [i for i, e in enumerate(targets) if e.action != ACTION_STOP] if stmt_rows: @@ -1130,9 +1244,21 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: loss = loss + slot_loss losses["slot"] = float(slot_loss.detach().cpu()) value_t = torch.tensor(values, device=device, dtype=out["value"].dtype) - value_loss = F.mse_loss(out["value"], value_t) - loss = loss + value_loss - losses["value"] = float(value_loss.detach().cpu()) + mask_t = torch.tensor(value_mask, device=device, dtype=torch.bool) + if bool(mask_t.any()): + value_loss = F.mse_loss(out["value"][:n_main][mask_t], value_t[mask_t]) + loss = loss + value_loss + losses["value"] = float(value_loss.detach().cpu()) + if pair_rows: + pair_loss = pairwise_progress_loss( + out["value"][torch.tensor([row for row, _ in pair_rows], device=device)], + out["value"][torch.tensor(pair_child_rows, device=device)], + margin=self.config.pairwise_progress_margin, + ) + loss = loss + pair_loss + losses["pairwise_progress"] = float(pair_loss.detach().cpu()) + losses["value_bounded"] = float(n_bounded) + losses["value_unknown_excluded"] = float(n_unknown_excluded) losses["skipped"] = float(skipped) self.last_training_metrics = losses return loss @@ -1444,7 +1570,11 @@ def from_checkpoint( "slm_training.models.checkpoint_migrate.migrate_tree_edit_checkpoint" ) tokenizer = OpenUITokenizer.load(path.with_suffix(".tokenizer.json")) - config = TreeEditDiffusionConfig(**payload["config"]) + config_payload = dict(payload["config"]) + # SLM-308: checkpoints written before value_label_mode existed were + # trained with mutation-count labels — preserve that behavior exactly. + config_payload.setdefault("value_label_mode", "mutation_count") + config = TreeEditDiffusionConfig(**config_payload) model = cls(tokenizer, config=config, device=device) model.load_state_dict(payload["state_dict"], strict=True) return model diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 0397d95b9..7a3a7c3bd 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -665,7 +665,7 @@ ] }, "evals.agentv": { - "version": "v3", + "version": "v4", "kind": "harness", "paths": [ "src/slm_training/evals/agentv.py", @@ -673,6 +673,11 @@ "tests/test_evals/test_agentv.py" ], "history": [ + { + "version": "v4", + "date": "2026-07-24", + "note": "Replace the optional Node LangSmith SDK with bounded native fetch so Vercel does not bundle CLI-only dependencies" + }, { "version": "v3", "date": "2026-07-24", @@ -4475,7 +4480,7 @@ ] }, "harness.experiments.slm299_edit_reachability": { - "version": "v2", + "version": "v3", "kind": "harness", "paths": [ "src/slm_training/harnesses/experiments/slm299_edit_reachability.py", @@ -4489,6 +4494,11 @@ "tests/test_models/test_tree_edit_diffusion.py" ], "history": [ + { + "version": "v3", + "date": "2026-07-24", + "note": "SLM-308: tree_edit_diffusion gains value_label_mode (bounded_distance default for new trainings, mutation_count parity for pre-field checkpoints), oracle-backed normalized cost-to-go value targets with UNKNOWN exclusion, and pairwise parent/improving-child progress loss; edit space and reachability machinery unchanged" + }, { "version": "v2", "date": "2026-07-24", @@ -4557,6 +4567,24 @@ } ] }, + "harness.experiments.slm308_distance_value": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm308_distance_oracle.py", + "scripts/run_slm308_distance_value.py", + "tests/test_harnesses/experiments/test_slm308_distance_value.py", + "docs/design/iter-slm308-distance-value-20260724.json", + "docs/design/iter-slm308-distance-value-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-308 LAR2-02 bounded reverse-BFS distance oracle (EXACT/BOUNDED/UNKNOWN) over canonical AST fingerprints, distance value supervision + pairwise progress ranking in the X22 tree-edit model, matched fixture experiment with preregistered thresholds" + } + ] + }, "harness.model_build.eval": { "version": "v54", "kind": "harness", diff --git a/tests/test_harnesses/experiments/test_slm308_distance_value.py b/tests/test_harnesses/experiments/test_slm308_distance_value.py new file mode 100644 index 000000000..472a39f7c --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm308_distance_value.py @@ -0,0 +1,357 @@ +"""SLM-308 (LAR2-02): distance oracle, value supervision, and matched-experiment tests.""" + +from __future__ import annotations + +import inspect + +import pytest +import torch + +import slm_training.harnesses.experiments.slm308_distance_oracle as oracle +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm308_distance_oracle import ( + DistanceKind, + clear_caches, + distance_to_target, + effective_distance, +) +from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + pairwise_progress_loss, + parse_statements, +) +from scripts.run_slm308_distance_value import ( + build_eval_states, + build_fixture_records, + build_report, + concordance, + render_markdown, + spearman, +) + +SEED = 'root = Stack([], "column")' +GOLD = 'root = Stack([cta], "column")\ncta = Button(":cta.label")' + + +@pytest.fixture() +def space(): + from slm_training.models.tree_edit_diffusion import TreeEditSpace + + return TreeEditSpace() + + +@pytest.fixture(autouse=True) +def _clean_oracle_caches(): + clear_caches() + yield + clear_caches() + + +# (a) EXACT / BOUNDED / UNKNOWN distinction --------------------------------- + + +def test_exact_distance_small_cases(space) -> None: + seed = parse_statements(SEED) + gold = parse_statements(GOLD) + label = distance_to_target( + seed, gold, space=space, inventory=[":cta.label"], node_budget=4 + ) + assert label.kind is DistanceKind.EXACT + assert label.distance == 1 + assert label.value_target(8) == pytest.approx(1.0 - 1 / 8) + # Identity is distance 0; the graph is undirected so reverse is exact too. + ident = distance_to_target(gold, gold, space=space, inventory=[":x"]) + assert ident.kind is DistanceKind.EXACT and ident.distance == 0 + rev = distance_to_target( + gold, seed, space=space, inventory=[":cta.label"], node_budget=4 + ) + assert rev.kind is DistanceKind.EXACT and rev.distance == 1 + + +def test_invariant_proof_is_bounded_without_finite_bounds(space) -> None: + seed = parse_statements(SEED) + bad = parse_statements('root = Stack([cta], "column")\ncta = Button(":zzz")') + label = distance_to_target(seed, bad, space=space, inventory=[":cta.label"]) + assert label.kind is DistanceKind.BOUNDED + assert label.reason == "invariant:needs_slot_rebind" + assert label.lo is None and label.hi is None + # Unbounded proofs are excluded from value supervision, never coerced. + assert label.value_target(8) is None + assert effective_distance(label) is None + + +def test_budget_exhaustion_is_explicit_unknown(space) -> None: + seed = parse_statements(SEED) + gold = parse_statements(GOLD) + # node_budget=1: the target layer expands, then the budget cuts layer 1 + # with a live frontier. UNKNOWN, never conflated with far/unreachable. + label = distance_to_target( + seed, gold, space=space, inventory=[":cta.label"], + max_depth=8, node_budget=1, + ) + # The seed itself is a depth-1 child of the target, so it IS reached; + # use a state that is not adjacent for the UNKNOWN check. + assert label.kind is DistanceKind.EXACT + far = parse_statements( + 'root = Stack([a, b], "column")\na = TextContent(":x")\nb = Button(":y")' + ) + # Both states' leaf slots must be in inventory, or the invariant proof + # (not the budget path) decides the label. + inv = [":x", ":y", ":cta.label"] + label = distance_to_target( + far, gold, space=space, inventory=inv, + max_depth=8, node_budget=1, + ) + assert label.kind is DistanceKind.UNKNOWN + assert label.reason == "budget" + assert label.value_target(8) is None + # A proven witness path upgrades UNKNOWN to BOUNDED(lo, hi) — the bounds + # are proofs (fully explored layers + witness), still never coerced exact. + bounded = distance_to_target( + far, gold, space=space, inventory=inv, + max_depth=8, node_budget=1, upper_bound_witness=3, + ) + assert bounded.kind is DistanceKind.BOUNDED + assert bounded.reason == "budget" + assert (bounded.lo, bounded.hi) == (2, 3) + assert bounded.value_target(8) == pytest.approx(1.0 - 2.5 / 8) + + +def test_depth_bound_is_bounded_not_unknown(space) -> None: + gold = parse_statements(GOLD) + far = parse_statements( + 'root = Stack([a, b], "column")\na = TextContent(":x")\nb = Button(":y")' + ) + # max_depth=0: nothing explored, budget intact, frontier still live. + label = distance_to_target( + far, gold, space=space, inventory=[":x", ":y", ":cta.label"], + max_depth=0, node_budget=1, upper_bound_witness=4, + ) + assert label.kind is DistanceKind.BOUNDED + assert label.reason == "depth_bound" + assert (label.lo, label.hi) == (1, 4) + + +# (b) cache identity ---------------------------------------------------------- + + +def test_cache_hit_and_version_change_misses(space, monkeypatch) -> None: + seed = parse_statements(SEED) + gold = parse_statements(GOLD) + kwargs = dict(space=space, inventory=[":cta.label"], node_budget=4) + first = distance_to_target(seed, gold, **kwargs) + assert not first.cache_hit + second = distance_to_target(seed, gold, **kwargs) + assert second.cache_hit and second.kind is first.kind + # Action-schema version change => guaranteed miss. + monkeypatch.setattr(oracle, "action_schema_version", lambda: "v999") + third = distance_to_target(seed, gold, **kwargs) + assert not third.cache_hit + # Grammar version change => guaranteed miss. + monkeypatch.setattr(oracle, "grammar_version", lambda: "sha256:deadbeef") + fourth = distance_to_target(seed, gold, **kwargs) + assert not fourth.cache_hit + + +def test_grammar_version_is_grammar_file_hash() -> None: + assert oracle.grammar_version().startswith("sha256:") + assert oracle.action_schema_version().startswith("v") + + +# (c) pairwise progress loss --------------------------------------------------- + + +def test_pairwise_progress_loss_improving_worsening_tie() -> None: + # Improving pair with margin satisfied: child scores >= parent + margin. + zero = pairwise_progress_loss( + torch.tensor([0.5]), torch.tensor([0.7]), margin=0.1 + ) + assert float(zero) == 0.0 + # Worsening pair (child below parent): full violation + margin. + loss = pairwise_progress_loss( + torch.tensor([0.6]), torch.tensor([0.4]), margin=0.1 + ) + assert float(loss) == pytest.approx(0.3) + # Tie inside the margin: only the margin remains. + tie = pairwise_progress_loss( + torch.tensor([0.5]), torch.tensor([0.5]), margin=0.1 + ) + assert float(tie) == pytest.approx(0.1) + # Empty pair set is a no-op zero, never an error. + empty = pairwise_progress_loss(torch.zeros(0), torch.zeros(0)) + assert float(empty) == 0.0 + # Gradient flows to both sides. + parent = torch.tensor([0.5], requires_grad=True) + child = torch.tensor([0.4], requires_grad=True) + pairwise_progress_loss(parent, child, margin=0.1).backward() + assert parent.grad is not None and child.grad is not None + + +# (d) no gold distance at inference -------------------------------------------- + +_RECORDS = [ + ExampleRecord( + id="r0", + prompt="hero with cta", + openui=GOLD, + placeholders=[":cta.label"], + split="train", + ), + ExampleRecord( + id="r1", + prompt="simple text", + openui='root = Stack([t], "column")\nt = TextContent(":body")', + placeholders=[":body"], + split="train", + ), +] + + +def _tiny_model(mode: str = "bounded_distance") -> TreeEditDiffusionModel: + config = TreeEditDiffusionConfig( + value_label_mode=mode, + d_model=32, + n_heads=2, + denoiser_layers=1, + context_layers=1, + beam_width=2, + expand_per_state=2, + max_search_steps=2, + max_chain=2, + ) + return TreeEditDiffusionModel.from_records(_RECORDS, config=config, device="cpu") + + +def test_decode_never_calls_the_oracle(monkeypatch) -> None: + # Runtime audit: any oracle call from the decode path fails loudly. + def _boom(*args, **kwargs): + raise AssertionError("oracle reached from inference path") + + monkeypatch.setattr(oracle, "distance_to_target", _boom) + model = _tiny_model() + text = model.generate("hero with cta") + assert isinstance(text, str) + # Static audit: the decode functions carry no oracle reference. + for fn_name in ("_decode_one", "_enumerate_edits", "_seed_state", "generate_batch_requests", "generate"): + source = inspect.getsource(getattr(TreeEditDiffusionModel, fn_name)) + assert "distance_to_target" not in source + assert "slm308" not in source + + +# (e) value supervision modes ---------------------------------------------------- + + +def test_training_loss_bounded_mode_uses_oracle_and_pairs() -> None: + model = _tiny_model("bounded_distance") + loss = model.training_loss(_RECORDS) + assert torch.isfinite(loss.detach()) + metrics = model.last_training_metrics + assert "value" in metrics + assert "value_bounded" in metrics + assert "value_unknown_excluded" in metrics + # Fixture chains are 1-2 edits from gold: at least one oracle-labeled + # value and one strictly-improving pair must exist. + assert metrics["value_bounded"] + 1 >= 1 + assert "pairwise_progress" in metrics + + +def test_training_loss_mutation_count_mode_matches_legacy() -> None: + model = _tiny_model("mutation_count") + loss = model.training_loss(_RECORDS) + assert torch.isfinite(loss.detach()) + metrics = model.last_training_metrics + # Legacy mode: every row carries a value, no oracle involvement, no pairs. + assert metrics["value_unknown_excluded"] == 0.0 + assert metrics["value_bounded"] == 0.0 + assert "pairwise_progress" not in metrics + + +def test_unknown_states_are_excluded_from_value_loss(monkeypatch) -> None: + model = _tiny_model("bounded_distance") + + class _Unknown: + kind = DistanceKind.UNKNOWN + + def value_target(self, max_depth: int): + return None + + monkeypatch.setattr( + TreeEditDiffusionModel, "_distance_label", lambda *a, **k: _Unknown() + ) + loss = model.training_loss(_RECORDS) + assert torch.isfinite(loss.detach()) + metrics = model.last_training_metrics + assert metrics["value_unknown_excluded"] >= 1.0 + + +# (f) checkpoint parity ----------------------------------------------------------- + + +def test_old_checkpoints_default_to_mutation_count(tmp_path) -> None: + model = _tiny_model("bounded_distance") + path = tmp_path / "ckpt.pt" + model.save(path) + payload = torch.load(path, map_location="cpu", weights_only=False) + assert payload["config"]["value_label_mode"] == "bounded_distance" + # Round-trip preserves the new mode. + loaded = TreeEditDiffusionModel.from_checkpoint(path) + assert loaded.config.value_label_mode == "bounded_distance" + # A format-2 checkpoint written before the field existed loads as + # mutation_count (behavior parity). + del payload["config"]["value_label_mode"] + torch.save(payload, path) + legacy = TreeEditDiffusionModel.from_checkpoint(path) + assert legacy.config.value_label_mode == "mutation_count" + + +def test_new_config_defaults_to_bounded_distance() -> None: + assert TreeEditDiffusionConfig().value_label_mode == "bounded_distance" + + +# (g) metric helpers + matched-experiment determinism ------------------------------ + + +def test_spearman_and_concordance() -> None: + assert spearman([1, 2, 3, 4], [1, 2, 3, 4]) == pytest.approx(1.0) + assert spearman([1, 2, 3, 4], [4, 3, 2, 1]) == pytest.approx(-1.0) + assert spearman([1, 1, 1], [1, 2, 3]) is None + assert concordance([0.1, 0.5, 0.9], [0.1, 0.5, 0.9]) == pytest.approx(1.0) + assert concordance([0.9, 0.5, 0.1], [0.1, 0.5, 0.9]) == pytest.approx(0.0) + + +def test_eval_states_are_deterministic(space) -> None: + records = build_fixture_records()[:2] + first = build_eval_states(records, space) + second = build_eval_states(records, space) + assert len(first) == len(second) > 0 + for a, b in zip(first, second): + assert a["origin"] == b["origin"] + assert a["witness"] == b["witness"] + from slm_training.models.tree_edit_diffusion import render_statements + + assert render_statements(a["statements"]) == render_statements(b["statements"]) + assert {e["origin"] for e in first} == {"near_gold", "seed_trajectory"} + + +@pytest.mark.training +def test_matched_experiment_is_deterministic() -> None: + import copy + + records = build_fixture_records()[:2] + first = build_report(records, steps=2, batch_size=2, seed=0) + second = build_report(records, steps=2, batch_size=2, seed=0) + md = render_markdown(first) + stripped = [] + for payload in (first, second): + view = copy.deepcopy(payload) + view.pop("version_stamp") + # Wall-clock oracle cost fields are evidence, not identity. + view["oracle_cost"].pop("wall_ms_total") + view["oracle_cost"].pop("ms_per_state") + stripped.append(view) + assert stripped[0] == stripped[1] + assert first["verdict"] in {"adopted", "rejected"} + assert first["preregistered_thresholds"]["beam_regret_improvement_min"] == 0.05 + md = render_markdown(first) + assert "Preregistered thresholds" in md From d8f35563d231e3b404b0fb0e213d68aee1b00a9d Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:11:37 -0500 Subject: [PATCH 06/18] SLM-310: align inverse-edit supervision with seed-originated demand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Proposal instrumentation: per enumerated candidate — action, factor score, applicability, rejection reason (23 machine-readable codes), budget consumption; deterministic order; additive evidence keys. - Distribution audit: training-target vs decode-demand action distributions by source/suite; dead-candidate rate ~0.98, applicable-ADD recall 0.10 baseline; preregistered reweighting rule. - corruption_action_distribution knob (gold corpus untouched; default off = historical uniform; parity tested) and stop_slot_accounting legacy|corrected arm (STOP consumes an expansion slot only when its frozen candidate is retained; deterministic, regression-tested). - Matched 2x2 (ADD-balanced x STOP arms, isolated levers): ADD target share 0.177 -> 0.328 (T1 +0.150 >= 0.10 both STOP arms), recall 0.10 -> 0.16, corrected STOP budget <= legacy; verdicts adopted per preregistered rules; loss reweighting deferred to preserve isolation. - 10 new tests; slm299 component v4; slm310 component v1. --- ...iter-slm310-action-alignment-20260724.json | 3792 +++++++++++++++++ .../iter-slm310-action-alignment-20260724.md | 38 + scripts/run_slm310_action_alignment.py | 481 +++ .../experiments/slm310_action_alignment.py | 330 ++ .../models/tree_edit_diffusion.py | 337 +- src/slm_training/resources/versions.json | 46 +- .../test_slm310_action_alignment.py | 405 ++ 7 files changed, 5343 insertions(+), 86 deletions(-) create mode 100644 docs/design/iter-slm310-action-alignment-20260724.json create mode 100644 docs/design/iter-slm310-action-alignment-20260724.md create mode 100644 scripts/run_slm310_action_alignment.py create mode 100644 src/slm_training/harnesses/experiments/slm310_action_alignment.py create mode 100644 tests/test_harnesses/experiments/test_slm310_action_alignment.py diff --git a/docs/design/iter-slm310-action-alignment-20260724.json b/docs/design/iter-slm310-action-alignment-20260724.json new file mode 100644 index 000000000..0dd364d00 --- /dev/null +++ b/docs/design/iter-slm310-action-alignment-20260724.json @@ -0,0 +1,3792 @@ +{ + "experiment": "slm310-action-alignment", + "issue": "SLM-310", + "question": "Does ADD-balanced corruption sampling close the train/demand action gap, and does corrected STOP-slot accounting improve search budget use \u2014 each lever isolated at fixture scale?", + "preregistered_thresholds": { + "t1_add_target_share_gain_min": 0.1, + "t2_valid_final_no_regression": true, + "t3_stop_budget_not_worse_and_valid_final_no_regression": true, + "t4_lever_isolation_structural": true, + "verdict_rule": "per lever: adopted iff its thresholds hold in both cells of the matched pair, else rejected; levers are never combined" + }, + "config": { + "steps": 8, + "batch_size": 4, + "seed": 0, + "audit_samples": 400, + "n_records": 8, + "value_label_mode": "mutation_count", + "add_balanced_distribution": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + }, + "cells": { + "cell_a": { + "corruption": null, + "stop": "legacy" + }, + "cell_b": { + "corruption": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + }, + "stop": "legacy" + }, + "cell_c": { + "corruption": null, + "stop": "corrected" + }, + "cell_d": { + "corruption": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + }, + "stop": "corrected" + } + } + }, + "cells": { + "cell_a": { + "levers": { + "corruption": null, + "stop": "legacy" + }, + "metrics": { + "training_targets": { + "samples": 400, + "n_targets": 378, + "counts": { + "STOP": 71, + "REPLACE": 42, + "ADD": 67, + "REMOVE": 69, + "ADD_CONTAINER": 5, + "REMOVE_CONTAINER": 69, + "INSERT_SUBTREE": 7, + "REPLACE_SUBTREE": 19, + "INSERT_STATEMENT": 0, + "REPLACE_STATEMENT": 7, + "BIND_PLACEHOLDER": 22 + }, + "shares": { + "STOP": 0.18783068783068782, + "REPLACE": 0.1111111111111111, + "ADD": 0.17724867724867724, + "REMOVE": 0.18253968253968253, + "ADD_CONTAINER": 0.013227513227513227, + "REMOVE_CONTAINER": 0.18253968253968253, + "INSERT_SUBTREE": 0.018518518518518517, + "REPLACE_SUBTREE": 0.05026455026455026, + "INSERT_STATEMENT": 0.0, + "REPLACE_STATEMENT": 0.018518518518518517, + "BIND_PLACEHOLDER": 0.0582010582010582 + }, + "inverse_action_weights": null + }, + "decode_demand": { + "overall": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 11092, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + }, + "by_suite": { + "train": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 0, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + } + }, + "by_source": { + "fixture_00": { + "visited": { + "BIND_PLACEHOLDER": 576, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 15, + "REPLACE": 3, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 297, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 9, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 185, + "invalid_result": 1, + "leaf_container_mismatch": 1, + "not_bindable": 252, + "not_canonical_subtree": 288, + "not_container": 15, + "not_leaf_component": 144, + "not_removable": 6, + "target_not_canonical": 2 + }, + "visited_total": 918, + "dead_total": 894, + "dead_candidate_rate": 0.9738562091503268, + "visited_shares": { + "BIND_PLACEHOLDER": 0.6274509803921569, + "INSERT_STATEMENT": 0.013071895424836602, + "REMOVE": 0.007625272331154684, + "REMOVE_CONTAINER": 0.016339869281045753, + "REPLACE": 0.0032679738562091504, + "REPLACE_STATEMENT": 0.002178649237472767, + "REPLACE_SUBTREE": 0.3235294117647059, + "STOP": 0.006535947712418301 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_01": { + "visited": { + "ADD": 2, + "BIND_PLACEHOLDER": 648, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 17, + "REMOVE": 3, + "REMOVE_CONTAINER": 14, + "REPLACE": 2, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 1022, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 3, + "REPLACE_SUBTREE": 230, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 17, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 7, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 407, + "invalid_result": 73, + "leaf_container_mismatch": 1, + "not_bindable": 432, + "not_canonical_subtree": 720, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 2, + "parent_or_comp_precondition": 17, + "target_not_canonical": 2 + }, + "visited_total": 1737, + "dead_total": 1701, + "dead_candidate_rate": 0.9792746113989638, + "visited_shares": { + "ADD": 0.0011514104778353484, + "BIND_PLACEHOLDER": 0.37305699481865284, + "INSERT_STATEMENT": 0.010362694300518135, + "INSERT_SUBTREE": 0.00978698906160046, + "REMOVE": 0.0017271157167530224, + "REMOVE_CONTAINER": 0.008059873344847437, + "REPLACE": 0.0011514104778353484, + "REPLACE_STATEMENT": 0.0011514104778353484, + "REPLACE_SUBTREE": 0.588370754173863, + "STOP": 0.0051813471502590676 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_02": { + "visited": { + "BIND_PLACEHOLDER": 504, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1431, + "dead_total": 1407, + "dead_candidate_rate": 0.9832285115303984, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3522012578616352, + "INSERT_STATEMENT": 0.008385744234800839, + "REMOVE": 0.004891684136967156, + "REMOVE_CONTAINER": 0.007686932215234102, + "REPLACE": 0.011879804332634521, + "REPLACE_STATEMENT": 0.0041928721174004195, + "REPLACE_SUBTREE": 0.6065688329839273, + "STOP": 0.0041928721174004195 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_03": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 8, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 72, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 75, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 8 + }, + "visited_total": 1397, + "dead_total": 1373, + "dead_candidate_rate": 0.9828203292770222, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3350035790980673, + "INSERT_STATEMENT": 0.008589835361488905, + "REMOVE": 0.005010737294201861, + "REMOVE_CONTAINER": 0.007874015748031496, + "REPLACE": 0.012168933428775949, + "REPLACE_STATEMENT": 0.00572655690765927, + "REPLACE_SUBTREE": 0.6213314244810307, + "STOP": 0.004294917680744453 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_04": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1395, + "dead_total": 1371, + "dead_candidate_rate": 0.9827956989247312, + "visited_shares": { + "BIND_PLACEHOLDER": 0.33548387096774196, + "INSERT_STATEMENT": 0.008602150537634409, + "REMOVE": 0.005017921146953405, + "REMOVE_CONTAINER": 0.007885304659498209, + "REPLACE": 0.012186379928315413, + "REPLACE_STATEMENT": 0.004301075268817204, + "REPLACE_SUBTREE": 0.6222222222222222, + "STOP": 0.004301075268817204 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_05": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 48, + "REMOVE": 4, + "REMOVE_CONTAINER": 9, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 9, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 50, + "target_not_canonical": 4 + }, + "visited_total": 1226, + "dead_total": 1194, + "dead_candidate_rate": 0.9738988580750407, + "visited_shares": { + "ADD": 0.0032626427406199023, + "BIND_PLACEHOLDER": 0.44045676998368677, + "INSERT_STATEMENT": 0.013050570962479609, + "INSERT_SUBTREE": 0.03915171288743882, + "REMOVE": 0.0032626427406199023, + "REMOVE_CONTAINER": 0.00734094616639478, + "REPLACE": 0.004893964110929853, + "REPLACE_STATEMENT": 0.0032626427406199023, + "REPLACE_SUBTREE": 0.47879282218597063, + "STOP": 0.0065252854812398045 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + }, + "fixture_06": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 720, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 33, + "REMOVE": 4, + "REMOVE_CONTAINER": 14, + "REPLACE": 5, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 1019, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 332, + "invalid_result": 38, + "leaf_container_mismatch": 2, + "not_bindable": 396, + "not_canonical_subtree": 864, + "not_container": 12, + "not_leaf_component": 108, + "not_removable": 3, + "parent_or_comp_precondition": 32, + "target_not_canonical": 4 + }, + "visited_total": 1827, + "dead_total": 1791, + "dead_candidate_rate": 0.9802955665024631, + "visited_shares": { + "ADD": 0.0005473453749315818, + "BIND_PLACEHOLDER": 0.39408866995073893, + "INSERT_STATEMENT": 0.009852216748768473, + "INSERT_SUBTREE": 0.0180623973727422, + "REMOVE": 0.0021893814997263274, + "REMOVE_CONTAINER": 0.007662835249042145, + "REPLACE": 0.002736726874657909, + "REPLACE_STATEMENT": 0.0021893814997263274, + "REPLACE_SUBTREE": 0.5577449370552818, + "STOP": 0.0049261083743842365 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_07": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 40, + "REMOVE": 4, + "REMOVE_CONTAINER": 10, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 10, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 42, + "target_not_canonical": 4 + }, + "visited_total": 1219, + "dead_total": 1187, + "dead_candidate_rate": 0.9737489745693191, + "visited_shares": { + "ADD": 0.003281378178835111, + "BIND_PLACEHOLDER": 0.44298605414273995, + "INSERT_STATEMENT": 0.013125512715340444, + "INSERT_SUBTREE": 0.03281378178835111, + "REMOVE": 0.003281378178835111, + "REMOVE_CONTAINER": 0.008203445447087777, + "REPLACE": 0.004922067268252666, + "REPLACE_STATEMENT": 0.003281378178835111, + "REPLACE_SUBTREE": 0.4815422477440525, + "STOP": 0.006562756357670222 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + } + } + }, + "calibration": { + "per_action": { + "STOP": { + "target_share": 0.18783068783068782, + "visited_share": 0.005201793721973094, + "gap": -0.18262889410871472 + }, + "REPLACE": { + "target_share": 0.1111111111111111, + "visited_share": 0.006547085201793722, + "gap": -0.10456402590931738 + }, + "ADD": { + "target_share": 0.17724867724867724, + "visited_share": 0.0009865470852017937, + "gap": -0.17626213016347544 + }, + "REMOVE": { + "target_share": 0.18253968253968253, + "visited_share": 0.0038565022421524664, + "gap": -0.17868318029753005 + }, + "ADD_CONTAINER": { + "target_share": 0.013227513227513227, + "visited_share": 0.0, + "gap": -0.013227513227513227 + }, + "REMOVE_CONTAINER": { + "target_share": 0.18253968253968253, + "visited_share": 0.008520179372197309, + "gap": -0.1740195031674852 + }, + "INSERT_SUBTREE": { + "target_share": 0.018518518518518517, + "visited_share": 0.012376681614349776, + "gap": -0.006141836904168741 + }, + "REPLACE_SUBTREE": { + "target_share": 0.05026455026455026, + "visited_share": 0.5485201793721973, + "gap": 0.4982556291076471 + }, + "INSERT_STATEMENT": { + "target_share": 0.0, + "visited_share": 0.010403587443946188, + "gap": 0.010403587443946188 + }, + "REPLACE_STATEMENT": { + "target_share": 0.018518518518518517, + "visited_share": 0.003228699551569507, + "gap": -0.01528981896694901 + }, + "BIND_PLACEHOLDER": { + "target_share": 0.0582010582010582, + "visited_share": 0.4003587443946188, + "gap": 0.34215768619356063 + } + }, + "mean_abs_deviation": 0.1546939823173007 + }, + "dead_candidate_rate": 0.9791928251121076, + "applicable_add_recall": 0.10344827586206896, + "verifier_calls": 11092, + "stop_budget_consumed": 58, + "stop_proposals_visited": 58, + "valid_final_rate": 1.0, + "n_records": 8, + "training_metrics": { + "action": 2.143547534942627, + "stmt": 2.8126392364501953, + "slot": 2.276818037033081, + "value": 0.21380357444286346, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + }, + "cell_b": { + "levers": { + "corruption": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + }, + "stop": "legacy" + }, + "metrics": { + "training_targets": { + "samples": 400, + "n_targets": 397, + "counts": { + "STOP": 79, + "REPLACE": 59, + "ADD": 130, + "REMOVE": 44, + "ADD_CONTAINER": 7, + "REMOVE_CONTAINER": 41, + "INSERT_SUBTREE": 12, + "REPLACE_SUBTREE": 9, + "INSERT_STATEMENT": 0, + "REPLACE_STATEMENT": 3, + "BIND_PLACEHOLDER": 13 + }, + "shares": { + "STOP": 0.19899244332493704, + "REPLACE": 0.1486146095717884, + "ADD": 0.327455919395466, + "REMOVE": 0.11083123425692695, + "ADD_CONTAINER": 0.017632241813602016, + "REMOVE_CONTAINER": 0.10327455919395466, + "INSERT_SUBTREE": 0.030226700251889168, + "REPLACE_SUBTREE": 0.022670025188916875, + "INSERT_STATEMENT": 0.0, + "REPLACE_STATEMENT": 0.007556675062972292, + "BIND_PLACEHOLDER": 0.0327455919395466 + }, + "inverse_action_weights": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + } + }, + "decode_demand": { + "overall": { + "visited": { + "ADD": 16, + "BIND_PLACEHOLDER": 5917, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 36, + "REMOVE": 84, + "REMOVE_CONTAINER": 150, + "REPLACE": 8, + "REPLACE_STATEMENT": 126, + "REPLACE_SUBTREE": 8144, + "STOP": 74 + }, + "applicable": { + "ADD": 12, + "BIND_PLACEHOLDER": 1453, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 22, + "REMOVE_CONTAINER": 20, + "REPLACE": 2, + "REPLACE_SUBTREE": 944, + "STOP": 74 + }, + "selected": { + "ADD": 8, + "BIND_PLACEHOLDER": 13, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 12, + "REMOVE_CONTAINER": 4, + "REPLACE": 2, + "REPLACE_SUBTREE": 31, + "STOP": 73 + }, + "rejection_reasons": { + "duplicate_state": 2384, + "invalid_result": 288, + "leaf_container_mismatch": 6, + "not_bindable": 4464, + "not_canonical_subtree": 6912, + "not_container": 130, + "not_removable": 62, + "parent_or_comp_precondition": 36, + "target_not_canonical": 126 + }, + "visited_total": 14703, + "dead_total": 14408, + "dead_candidate_rate": 0.979936067469224, + "visited_shares": { + "ADD": 0.0010882132898048017, + "BIND_PLACEHOLDER": 0.40243487723593824, + "INSERT_STATEMENT": 0.010065972930694417, + "INSERT_SUBTREE": 0.002448479902060804, + "REMOVE": 0.005713119771475209, + "REMOVE_CONTAINER": 0.010201999591920016, + "REPLACE": 0.0005441066449024008, + "REPLACE_STATEMENT": 0.008569679657212814, + "REPLACE_SUBTREE": 0.5539005645106441, + "STOP": 0.005032986465347208 + }, + "verifier_calls": 14629, + "states_with_applicable_add": 74, + "states_recalled": 12, + "applicable_add_recall": 0.16216216216216217 + }, + "by_suite": { + "train": { + "visited": { + "ADD": 16, + "BIND_PLACEHOLDER": 5917, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 36, + "REMOVE": 84, + "REMOVE_CONTAINER": 150, + "REPLACE": 8, + "REPLACE_STATEMENT": 126, + "REPLACE_SUBTREE": 8144, + "STOP": 74 + }, + "applicable": { + "ADD": 12, + "BIND_PLACEHOLDER": 1453, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 22, + "REMOVE_CONTAINER": 20, + "REPLACE": 2, + "REPLACE_SUBTREE": 944, + "STOP": 74 + }, + "selected": { + "ADD": 8, + "BIND_PLACEHOLDER": 13, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 12, + "REMOVE_CONTAINER": 4, + "REPLACE": 2, + "REPLACE_SUBTREE": 31, + "STOP": 73 + }, + "rejection_reasons": { + "duplicate_state": 2384, + "invalid_result": 288, + "leaf_container_mismatch": 6, + "not_bindable": 4464, + "not_canonical_subtree": 6912, + "not_container": 130, + "not_removable": 62, + "parent_or_comp_precondition": 36, + "target_not_canonical": 126 + }, + "visited_total": 14703, + "dead_total": 14408, + "dead_candidate_rate": 0.979936067469224, + "visited_shares": { + "ADD": 0.0010882132898048017, + "BIND_PLACEHOLDER": 0.40243487723593824, + "INSERT_STATEMENT": 0.010065972930694417, + "INSERT_SUBTREE": 0.002448479902060804, + "REMOVE": 0.005713119771475209, + "REMOVE_CONTAINER": 0.010201999591920016, + "REPLACE": 0.0005441066449024008, + "REPLACE_STATEMENT": 0.008569679657212814, + "REPLACE_SUBTREE": 0.5539005645106441, + "STOP": 0.005032986465347208 + }, + "verifier_calls": 0, + "states_with_applicable_add": 74, + "states_recalled": 12, + "applicable_add_recall": 0.16216216216216217 + } + }, + "by_source": { + "fixture_00": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 688, + "INSERT_STATEMENT": 16, + "REMOVE": 13, + "REMOVE_CONTAINER": 17, + "REPLACE": 1, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 293, + "STOP": 8 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 112, + "INSERT_STATEMENT": 16, + "REMOVE": 5, + "REPLACE_SUBTREE": 5, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 4, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE_SUBTREE": 2, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 115, + "leaf_container_mismatch": 1, + "not_bindable": 576, + "not_canonical_subtree": 288, + "not_container": 17, + "not_removable": 8, + "target_not_canonical": 16 + }, + "visited_total": 1053, + "dead_total": 1021, + "dead_candidate_rate": 0.9696106362773029, + "visited_shares": { + "ADD": 0.000949667616334283, + "BIND_PLACEHOLDER": 0.6533713200379867, + "INSERT_STATEMENT": 0.015194681861348529, + "REMOVE": 0.012345679012345678, + "REMOVE_CONTAINER": 0.016144349477682812, + "REPLACE": 0.000949667616334283, + "REPLACE_STATEMENT": 0.015194681861348529, + "REPLACE_SUBTREE": 0.27825261158594494, + "STOP": 0.007597340930674264 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 1, + "applicable_add_recall": 0.125 + }, + "fixture_01": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 21, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 16, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2556, + "dead_total": 2512, + "dead_candidate_rate": 0.9827856025039123, + "visited_shares": { + "ADD": 0.0011737089201877935, + "BIND_PLACEHOLDER": 0.3380281690140845, + "INSERT_STATEMENT": 0.008607198748043818, + "INSERT_SUBTREE": 0.0035211267605633804, + "REMOVE": 0.003912363067292645, + "REMOVE_CONTAINER": 0.008215962441314555, + "REPLACE_STATEMENT": 0.006259780907668232, + "REPLACE_SUBTREE": 0.6259780907668232, + "STOP": 0.004303599374021909 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_02": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 544, + "INSERT_STATEMENT": 14, + "REMOVE": 10, + "REMOVE_CONTAINER": 14, + "REPLACE": 3, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 576, + "STOP": 7 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 40, + "INSERT_STATEMENT": 14, + "REMOVE": 3, + "REPLACE": 1, + "STOP": 7 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 4, + "INSERT_STATEMENT": 14, + "REMOVE": 1, + "REPLACE": 1, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 38, + "leaf_container_mismatch": 2, + "not_bindable": 504, + "not_canonical_subtree": 576, + "not_container": 14, + "not_removable": 7, + "target_not_canonical": 16 + }, + "visited_total": 1185, + "dead_total": 1157, + "dead_candidate_rate": 0.9763713080168777, + "visited_shares": { + "ADD": 0.0008438818565400844, + "BIND_PLACEHOLDER": 0.45907172995780593, + "INSERT_STATEMENT": 0.01181434599156118, + "REMOVE": 0.008438818565400843, + "REMOVE_CONTAINER": 0.01181434599156118, + "REPLACE": 0.002531645569620253, + "REPLACE_STATEMENT": 0.01350210970464135, + "REPLACE_SUBTREE": 0.4860759493670886, + "STOP": 0.00590717299578059 + }, + "verifier_calls": 0, + "states_with_applicable_add": 7, + "states_recalled": 1, + "applicable_add_recall": 0.14285714285714285 + }, + "fixture_03": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 579, + "INSERT_STATEMENT": 14, + "REMOVE": 9, + "REMOVE_CONTAINER": 15, + "REPLACE": 3, + "REPLACE_STATEMENT": 14, + "REPLACE_SUBTREE": 578, + "STOP": 7 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 75, + "INSERT_STATEMENT": 14, + "REMOVE": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 2, + "STOP": 7 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 3, + "INSERT_STATEMENT": 14, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 1, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 74, + "leaf_container_mismatch": 2, + "not_bindable": 504, + "not_canonical_subtree": 576, + "not_container": 15, + "not_removable": 7, + "target_not_canonical": 14 + }, + "visited_total": 1220, + "dead_total": 1192, + "dead_candidate_rate": 0.9770491803278688, + "visited_shares": { + "ADD": 0.000819672131147541, + "BIND_PLACEHOLDER": 0.4745901639344262, + "INSERT_STATEMENT": 0.011475409836065573, + "REMOVE": 0.007377049180327869, + "REMOVE_CONTAINER": 0.012295081967213115, + "REPLACE": 0.002459016393442623, + "REPLACE_STATEMENT": 0.011475409836065573, + "REPLACE_SUBTREE": 0.4737704918032787, + "STOP": 0.005737704918032787 + }, + "verifier_calls": 0, + "states_with_applicable_add": 7, + "states_recalled": 1, + "applicable_add_recall": 0.14285714285714285 + }, + "fixture_04": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 650, + "INSERT_STATEMENT": 16, + "REMOVE": 12, + "REMOVE_CONTAINER": 18, + "REPLACE": 1, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 297, + "STOP": 8 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 74, + "INSERT_STATEMENT": 16, + "REMOVE": 4, + "REPLACE_SUBTREE": 9, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 2, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 81, + "leaf_container_mismatch": 1, + "not_bindable": 576, + "not_canonical_subtree": 288, + "not_container": 18, + "not_removable": 8, + "target_not_canonical": 16 + }, + "visited_total": 1019, + "dead_total": 988, + "dead_candidate_rate": 0.9695780176643768, + "visited_shares": { + "ADD": 0.0009813542688910696, + "BIND_PLACEHOLDER": 0.6378802747791953, + "INSERT_STATEMENT": 0.015701668302257114, + "REMOVE": 0.011776251226692836, + "REMOVE_CONTAINER": 0.017664376840039256, + "REPLACE": 0.0009813542688910696, + "REPLACE_STATEMENT": 0.015701668302257114, + "REPLACE_SUBTREE": 0.2914622178606477, + "STOP": 0.007850834151128557 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 1, + "applicable_add_recall": 0.125 + }, + "fixture_05": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 22, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 17, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2557, + "dead_total": 2513, + "dead_candidate_rate": 0.9827923347673054, + "visited_shares": { + "ADD": 0.0011732499022291747, + "BIND_PLACEHOLDER": 0.3378959718420024, + "INSERT_STATEMENT": 0.008603832616347283, + "INSERT_SUBTREE": 0.0035197497066875244, + "REMOVE": 0.003910833007430583, + "REMOVE_CONTAINER": 0.008603832616347283, + "REPLACE_STATEMENT": 0.006257332811888932, + "REPLACE_SUBTREE": 0.6257332811888933, + "STOP": 0.004301916308173641 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_06": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 22, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 17, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2557, + "dead_total": 2513, + "dead_candidate_rate": 0.9827923347673054, + "visited_shares": { + "ADD": 0.0011732499022291747, + "BIND_PLACEHOLDER": 0.3378959718420024, + "INSERT_STATEMENT": 0.008603832616347283, + "INSERT_SUBTREE": 0.0035197497066875244, + "REMOVE": 0.003910833007430583, + "REMOVE_CONTAINER": 0.008603832616347283, + "REPLACE_STATEMENT": 0.006257332811888932, + "REPLACE_SUBTREE": 0.6257332811888933, + "STOP": 0.004301916308173641 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_07": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 21, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 16, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2556, + "dead_total": 2512, + "dead_candidate_rate": 0.9827856025039123, + "visited_shares": { + "ADD": 0.0011737089201877935, + "BIND_PLACEHOLDER": 0.3380281690140845, + "INSERT_STATEMENT": 0.008607198748043818, + "INSERT_SUBTREE": 0.0035211267605633804, + "REMOVE": 0.003912363067292645, + "REMOVE_CONTAINER": 0.008215962441314555, + "REPLACE_STATEMENT": 0.006259780907668232, + "REPLACE_SUBTREE": 0.6259780907668232, + "STOP": 0.004303599374021909 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + } + } + }, + "calibration": { + "per_action": { + "STOP": { + "target_share": 0.19899244332493704, + "visited_share": 0.005032986465347208, + "gap": -0.19395945685958982 + }, + "REPLACE": { + "target_share": 0.1486146095717884, + "visited_share": 0.0005441066449024008, + "gap": -0.148070502926886 + }, + "ADD": { + "target_share": 0.327455919395466, + "visited_share": 0.0010882132898048017, + "gap": -0.3263677061056612 + }, + "REMOVE": { + "target_share": 0.11083123425692695, + "visited_share": 0.005713119771475209, + "gap": -0.10511811448545175 + }, + "ADD_CONTAINER": { + "target_share": 0.017632241813602016, + "visited_share": 0.0, + "gap": -0.017632241813602016 + }, + "REMOVE_CONTAINER": { + "target_share": 0.10327455919395466, + "visited_share": 0.010201999591920016, + "gap": -0.09307255960203464 + }, + "INSERT_SUBTREE": { + "target_share": 0.030226700251889168, + "visited_share": 0.002448479902060804, + "gap": -0.027778220349828363 + }, + "REPLACE_SUBTREE": { + "target_share": 0.022670025188916875, + "visited_share": 0.5539005645106441, + "gap": 0.5312305393217271 + }, + "INSERT_STATEMENT": { + "target_share": 0.0, + "visited_share": 0.010065972930694417, + "gap": 0.010065972930694417 + }, + "REPLACE_STATEMENT": { + "target_share": 0.007556675062972292, + "visited_share": 0.008569679657212814, + "gap": 0.0010130045942405219 + }, + "BIND_PLACEHOLDER": { + "target_share": 0.0327455919395466, + "visited_share": 0.40243487723593824, + "gap": 0.36968928529639167 + } + }, + "mean_abs_deviation": 0.16581796402600976 + }, + "dead_candidate_rate": 0.979936067469224, + "applicable_add_recall": 0.16216216216216217, + "verifier_calls": 14629, + "stop_budget_consumed": 74, + "stop_proposals_visited": 74, + "valid_final_rate": 1.0, + "n_records": 8, + "training_metrics": { + "action": 2.112842082977295, + "stmt": 2.703423023223877, + "comp": 2.7941222190856934, + "slot": 2.7572789192199707, + "value": 0.16673196852207184, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + }, + "cell_c": { + "levers": { + "corruption": null, + "stop": "corrected" + }, + "metrics": { + "training_targets": { + "samples": 400, + "n_targets": 378, + "counts": { + "STOP": 71, + "REPLACE": 42, + "ADD": 67, + "REMOVE": 69, + "ADD_CONTAINER": 5, + "REMOVE_CONTAINER": 69, + "INSERT_SUBTREE": 7, + "REPLACE_SUBTREE": 19, + "INSERT_STATEMENT": 0, + "REPLACE_STATEMENT": 7, + "BIND_PLACEHOLDER": 22 + }, + "shares": { + "STOP": 0.18783068783068782, + "REPLACE": 0.1111111111111111, + "ADD": 0.17724867724867724, + "REMOVE": 0.18253968253968253, + "ADD_CONTAINER": 0.013227513227513227, + "REMOVE_CONTAINER": 0.18253968253968253, + "INSERT_SUBTREE": 0.018518518518518517, + "REPLACE_SUBTREE": 0.05026455026455026, + "INSERT_STATEMENT": 0.0, + "REPLACE_STATEMENT": 0.018518518518518517, + "BIND_PLACEHOLDER": 0.0582010582010582 + }, + "inverse_action_weights": null + }, + "decode_demand": { + "overall": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 11092, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + }, + "by_suite": { + "train": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 0, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + } + }, + "by_source": { + "fixture_00": { + "visited": { + "BIND_PLACEHOLDER": 576, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 15, + "REPLACE": 3, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 297, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 9, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 185, + "invalid_result": 1, + "leaf_container_mismatch": 1, + "not_bindable": 252, + "not_canonical_subtree": 288, + "not_container": 15, + "not_leaf_component": 144, + "not_removable": 6, + "target_not_canonical": 2 + }, + "visited_total": 918, + "dead_total": 894, + "dead_candidate_rate": 0.9738562091503268, + "visited_shares": { + "BIND_PLACEHOLDER": 0.6274509803921569, + "INSERT_STATEMENT": 0.013071895424836602, + "REMOVE": 0.007625272331154684, + "REMOVE_CONTAINER": 0.016339869281045753, + "REPLACE": 0.0032679738562091504, + "REPLACE_STATEMENT": 0.002178649237472767, + "REPLACE_SUBTREE": 0.3235294117647059, + "STOP": 0.006535947712418301 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_01": { + "visited": { + "ADD": 2, + "BIND_PLACEHOLDER": 648, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 17, + "REMOVE": 3, + "REMOVE_CONTAINER": 14, + "REPLACE": 2, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 1022, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 3, + "REPLACE_SUBTREE": 230, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 17, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 7, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 407, + "invalid_result": 73, + "leaf_container_mismatch": 1, + "not_bindable": 432, + "not_canonical_subtree": 720, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 2, + "parent_or_comp_precondition": 17, + "target_not_canonical": 2 + }, + "visited_total": 1737, + "dead_total": 1701, + "dead_candidate_rate": 0.9792746113989638, + "visited_shares": { + "ADD": 0.0011514104778353484, + "BIND_PLACEHOLDER": 0.37305699481865284, + "INSERT_STATEMENT": 0.010362694300518135, + "INSERT_SUBTREE": 0.00978698906160046, + "REMOVE": 0.0017271157167530224, + "REMOVE_CONTAINER": 0.008059873344847437, + "REPLACE": 0.0011514104778353484, + "REPLACE_STATEMENT": 0.0011514104778353484, + "REPLACE_SUBTREE": 0.588370754173863, + "STOP": 0.0051813471502590676 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_02": { + "visited": { + "BIND_PLACEHOLDER": 504, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1431, + "dead_total": 1407, + "dead_candidate_rate": 0.9832285115303984, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3522012578616352, + "INSERT_STATEMENT": 0.008385744234800839, + "REMOVE": 0.004891684136967156, + "REMOVE_CONTAINER": 0.007686932215234102, + "REPLACE": 0.011879804332634521, + "REPLACE_STATEMENT": 0.0041928721174004195, + "REPLACE_SUBTREE": 0.6065688329839273, + "STOP": 0.0041928721174004195 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_03": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 8, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 72, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 75, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 8 + }, + "visited_total": 1397, + "dead_total": 1373, + "dead_candidate_rate": 0.9828203292770222, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3350035790980673, + "INSERT_STATEMENT": 0.008589835361488905, + "REMOVE": 0.005010737294201861, + "REMOVE_CONTAINER": 0.007874015748031496, + "REPLACE": 0.012168933428775949, + "REPLACE_STATEMENT": 0.00572655690765927, + "REPLACE_SUBTREE": 0.6213314244810307, + "STOP": 0.004294917680744453 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_04": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1395, + "dead_total": 1371, + "dead_candidate_rate": 0.9827956989247312, + "visited_shares": { + "BIND_PLACEHOLDER": 0.33548387096774196, + "INSERT_STATEMENT": 0.008602150537634409, + "REMOVE": 0.005017921146953405, + "REMOVE_CONTAINER": 0.007885304659498209, + "REPLACE": 0.012186379928315413, + "REPLACE_STATEMENT": 0.004301075268817204, + "REPLACE_SUBTREE": 0.6222222222222222, + "STOP": 0.004301075268817204 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_05": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 48, + "REMOVE": 4, + "REMOVE_CONTAINER": 9, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 9, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 50, + "target_not_canonical": 4 + }, + "visited_total": 1226, + "dead_total": 1194, + "dead_candidate_rate": 0.9738988580750407, + "visited_shares": { + "ADD": 0.0032626427406199023, + "BIND_PLACEHOLDER": 0.44045676998368677, + "INSERT_STATEMENT": 0.013050570962479609, + "INSERT_SUBTREE": 0.03915171288743882, + "REMOVE": 0.0032626427406199023, + "REMOVE_CONTAINER": 0.00734094616639478, + "REPLACE": 0.004893964110929853, + "REPLACE_STATEMENT": 0.0032626427406199023, + "REPLACE_SUBTREE": 0.47879282218597063, + "STOP": 0.0065252854812398045 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + }, + "fixture_06": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 720, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 33, + "REMOVE": 4, + "REMOVE_CONTAINER": 14, + "REPLACE": 5, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 1019, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 332, + "invalid_result": 38, + "leaf_container_mismatch": 2, + "not_bindable": 396, + "not_canonical_subtree": 864, + "not_container": 12, + "not_leaf_component": 108, + "not_removable": 3, + "parent_or_comp_precondition": 32, + "target_not_canonical": 4 + }, + "visited_total": 1827, + "dead_total": 1791, + "dead_candidate_rate": 0.9802955665024631, + "visited_shares": { + "ADD": 0.0005473453749315818, + "BIND_PLACEHOLDER": 0.39408866995073893, + "INSERT_STATEMENT": 0.009852216748768473, + "INSERT_SUBTREE": 0.0180623973727422, + "REMOVE": 0.0021893814997263274, + "REMOVE_CONTAINER": 0.007662835249042145, + "REPLACE": 0.002736726874657909, + "REPLACE_STATEMENT": 0.0021893814997263274, + "REPLACE_SUBTREE": 0.5577449370552818, + "STOP": 0.0049261083743842365 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_07": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 40, + "REMOVE": 4, + "REMOVE_CONTAINER": 10, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 10, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 42, + "target_not_canonical": 4 + }, + "visited_total": 1219, + "dead_total": 1187, + "dead_candidate_rate": 0.9737489745693191, + "visited_shares": { + "ADD": 0.003281378178835111, + "BIND_PLACEHOLDER": 0.44298605414273995, + "INSERT_STATEMENT": 0.013125512715340444, + "INSERT_SUBTREE": 0.03281378178835111, + "REMOVE": 0.003281378178835111, + "REMOVE_CONTAINER": 0.008203445447087777, + "REPLACE": 0.004922067268252666, + "REPLACE_STATEMENT": 0.003281378178835111, + "REPLACE_SUBTREE": 0.4815422477440525, + "STOP": 0.006562756357670222 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + } + } + }, + "calibration": { + "per_action": { + "STOP": { + "target_share": 0.18783068783068782, + "visited_share": 0.005201793721973094, + "gap": -0.18262889410871472 + }, + "REPLACE": { + "target_share": 0.1111111111111111, + "visited_share": 0.006547085201793722, + "gap": -0.10456402590931738 + }, + "ADD": { + "target_share": 0.17724867724867724, + "visited_share": 0.0009865470852017937, + "gap": -0.17626213016347544 + }, + "REMOVE": { + "target_share": 0.18253968253968253, + "visited_share": 0.0038565022421524664, + "gap": -0.17868318029753005 + }, + "ADD_CONTAINER": { + "target_share": 0.013227513227513227, + "visited_share": 0.0, + "gap": -0.013227513227513227 + }, + "REMOVE_CONTAINER": { + "target_share": 0.18253968253968253, + "visited_share": 0.008520179372197309, + "gap": -0.1740195031674852 + }, + "INSERT_SUBTREE": { + "target_share": 0.018518518518518517, + "visited_share": 0.012376681614349776, + "gap": -0.006141836904168741 + }, + "REPLACE_SUBTREE": { + "target_share": 0.05026455026455026, + "visited_share": 0.5485201793721973, + "gap": 0.4982556291076471 + }, + "INSERT_STATEMENT": { + "target_share": 0.0, + "visited_share": 0.010403587443946188, + "gap": 0.010403587443946188 + }, + "REPLACE_STATEMENT": { + "target_share": 0.018518518518518517, + "visited_share": 0.003228699551569507, + "gap": -0.01528981896694901 + }, + "BIND_PLACEHOLDER": { + "target_share": 0.0582010582010582, + "visited_share": 0.4003587443946188, + "gap": 0.34215768619356063 + } + }, + "mean_abs_deviation": 0.1546939823173007 + }, + "dead_candidate_rate": 0.9791928251121076, + "applicable_add_recall": 0.10344827586206896, + "verifier_calls": 11092, + "stop_budget_consumed": 58, + "stop_proposals_visited": 58, + "valid_final_rate": 1.0, + "n_records": 8, + "training_metrics": { + "action": 2.143547534942627, + "stmt": 2.8126392364501953, + "slot": 2.276818037033081, + "value": 0.21380357444286346, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + }, + "cell_d": { + "levers": { + "corruption": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + }, + "stop": "corrected" + }, + "metrics": { + "training_targets": { + "samples": 400, + "n_targets": 397, + "counts": { + "STOP": 79, + "REPLACE": 59, + "ADD": 130, + "REMOVE": 44, + "ADD_CONTAINER": 7, + "REMOVE_CONTAINER": 41, + "INSERT_SUBTREE": 12, + "REPLACE_SUBTREE": 9, + "INSERT_STATEMENT": 0, + "REPLACE_STATEMENT": 3, + "BIND_PLACEHOLDER": 13 + }, + "shares": { + "STOP": 0.19899244332493704, + "REPLACE": 0.1486146095717884, + "ADD": 0.327455919395466, + "REMOVE": 0.11083123425692695, + "ADD_CONTAINER": 0.017632241813602016, + "REMOVE_CONTAINER": 0.10327455919395466, + "INSERT_SUBTREE": 0.030226700251889168, + "REPLACE_SUBTREE": 0.022670025188916875, + "INSERT_STATEMENT": 0.0, + "REPLACE_STATEMENT": 0.007556675062972292, + "BIND_PLACEHOLDER": 0.0327455919395466 + }, + "inverse_action_weights": { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0 + } + }, + "decode_demand": { + "overall": { + "visited": { + "ADD": 16, + "BIND_PLACEHOLDER": 5917, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 36, + "REMOVE": 84, + "REMOVE_CONTAINER": 150, + "REPLACE": 8, + "REPLACE_STATEMENT": 126, + "REPLACE_SUBTREE": 8145, + "STOP": 74 + }, + "applicable": { + "ADD": 12, + "BIND_PLACEHOLDER": 1453, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 22, + "REMOVE_CONTAINER": 20, + "REPLACE": 2, + "REPLACE_SUBTREE": 945, + "STOP": 74 + }, + "selected": { + "ADD": 8, + "BIND_PLACEHOLDER": 13, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 12, + "REMOVE_CONTAINER": 4, + "REPLACE": 2, + "REPLACE_SUBTREE": 32, + "STOP": 73 + }, + "rejection_reasons": { + "duplicate_state": 2384, + "invalid_result": 288, + "leaf_container_mismatch": 6, + "not_bindable": 4464, + "not_canonical_subtree": 6912, + "not_container": 130, + "not_removable": 62, + "parent_or_comp_precondition": 36, + "target_not_canonical": 126 + }, + "visited_total": 14704, + "dead_total": 14408, + "dead_candidate_rate": 0.9798694232861807, + "visited_shares": { + "ADD": 0.001088139281828074, + "BIND_PLACEHOLDER": 0.4024075081610446, + "INSERT_STATEMENT": 0.010065288356909684, + "INSERT_SUBTREE": 0.0024483133841131663, + "REMOVE": 0.005712731229597388, + "REMOVE_CONTAINER": 0.010201305767138194, + "REPLACE": 0.000544069640914037, + "REPLACE_STATEMENT": 0.008569096844396083, + "REPLACE_SUBTREE": 0.5539309031556039, + "STOP": 0.005032644178454842 + }, + "verifier_calls": 14630, + "states_with_applicable_add": 74, + "states_recalled": 12, + "applicable_add_recall": 0.16216216216216217 + }, + "by_suite": { + "train": { + "visited": { + "ADD": 16, + "BIND_PLACEHOLDER": 5917, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 36, + "REMOVE": 84, + "REMOVE_CONTAINER": 150, + "REPLACE": 8, + "REPLACE_STATEMENT": 126, + "REPLACE_SUBTREE": 8145, + "STOP": 74 + }, + "applicable": { + "ADD": 12, + "BIND_PLACEHOLDER": 1453, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 22, + "REMOVE_CONTAINER": 20, + "REPLACE": 2, + "REPLACE_SUBTREE": 945, + "STOP": 74 + }, + "selected": { + "ADD": 8, + "BIND_PLACEHOLDER": 13, + "INSERT_STATEMENT": 148, + "INSERT_SUBTREE": 4, + "REMOVE": 12, + "REMOVE_CONTAINER": 4, + "REPLACE": 2, + "REPLACE_SUBTREE": 32, + "STOP": 73 + }, + "rejection_reasons": { + "duplicate_state": 2384, + "invalid_result": 288, + "leaf_container_mismatch": 6, + "not_bindable": 4464, + "not_canonical_subtree": 6912, + "not_container": 130, + "not_removable": 62, + "parent_or_comp_precondition": 36, + "target_not_canonical": 126 + }, + "visited_total": 14704, + "dead_total": 14408, + "dead_candidate_rate": 0.9798694232861807, + "visited_shares": { + "ADD": 0.001088139281828074, + "BIND_PLACEHOLDER": 0.4024075081610446, + "INSERT_STATEMENT": 0.010065288356909684, + "INSERT_SUBTREE": 0.0024483133841131663, + "REMOVE": 0.005712731229597388, + "REMOVE_CONTAINER": 0.010201305767138194, + "REPLACE": 0.000544069640914037, + "REPLACE_STATEMENT": 0.008569096844396083, + "REPLACE_SUBTREE": 0.5539309031556039, + "STOP": 0.005032644178454842 + }, + "verifier_calls": 0, + "states_with_applicable_add": 74, + "states_recalled": 12, + "applicable_add_recall": 0.16216216216216217 + } + }, + "by_source": { + "fixture_00": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 688, + "INSERT_STATEMENT": 16, + "REMOVE": 13, + "REMOVE_CONTAINER": 17, + "REPLACE": 1, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 293, + "STOP": 8 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 112, + "INSERT_STATEMENT": 16, + "REMOVE": 5, + "REPLACE_SUBTREE": 5, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 4, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE_SUBTREE": 2, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 115, + "leaf_container_mismatch": 1, + "not_bindable": 576, + "not_canonical_subtree": 288, + "not_container": 17, + "not_removable": 8, + "target_not_canonical": 16 + }, + "visited_total": 1053, + "dead_total": 1021, + "dead_candidate_rate": 0.9696106362773029, + "visited_shares": { + "ADD": 0.000949667616334283, + "BIND_PLACEHOLDER": 0.6533713200379867, + "INSERT_STATEMENT": 0.015194681861348529, + "REMOVE": 0.012345679012345678, + "REMOVE_CONTAINER": 0.016144349477682812, + "REPLACE": 0.000949667616334283, + "REPLACE_STATEMENT": 0.015194681861348529, + "REPLACE_SUBTREE": 0.27825261158594494, + "STOP": 0.007597340930674264 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 1, + "applicable_add_recall": 0.125 + }, + "fixture_01": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 21, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 16, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2556, + "dead_total": 2512, + "dead_candidate_rate": 0.9827856025039123, + "visited_shares": { + "ADD": 0.0011737089201877935, + "BIND_PLACEHOLDER": 0.3380281690140845, + "INSERT_STATEMENT": 0.008607198748043818, + "INSERT_SUBTREE": 0.0035211267605633804, + "REMOVE": 0.003912363067292645, + "REMOVE_CONTAINER": 0.008215962441314555, + "REPLACE_STATEMENT": 0.006259780907668232, + "REPLACE_SUBTREE": 0.6259780907668232, + "STOP": 0.004303599374021909 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_02": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 544, + "INSERT_STATEMENT": 14, + "REMOVE": 10, + "REMOVE_CONTAINER": 14, + "REPLACE": 3, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 576, + "STOP": 7 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 40, + "INSERT_STATEMENT": 14, + "REMOVE": 3, + "REPLACE": 1, + "STOP": 7 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 4, + "INSERT_STATEMENT": 14, + "REMOVE": 1, + "REPLACE": 1, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 38, + "leaf_container_mismatch": 2, + "not_bindable": 504, + "not_canonical_subtree": 576, + "not_container": 14, + "not_removable": 7, + "target_not_canonical": 16 + }, + "visited_total": 1185, + "dead_total": 1157, + "dead_candidate_rate": 0.9763713080168777, + "visited_shares": { + "ADD": 0.0008438818565400844, + "BIND_PLACEHOLDER": 0.45907172995780593, + "INSERT_STATEMENT": 0.01181434599156118, + "REMOVE": 0.008438818565400843, + "REMOVE_CONTAINER": 0.01181434599156118, + "REPLACE": 0.002531645569620253, + "REPLACE_STATEMENT": 0.01350210970464135, + "REPLACE_SUBTREE": 0.4860759493670886, + "STOP": 0.00590717299578059 + }, + "verifier_calls": 0, + "states_with_applicable_add": 7, + "states_recalled": 1, + "applicable_add_recall": 0.14285714285714285 + }, + "fixture_03": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 579, + "INSERT_STATEMENT": 14, + "REMOVE": 9, + "REMOVE_CONTAINER": 15, + "REPLACE": 3, + "REPLACE_STATEMENT": 14, + "REPLACE_SUBTREE": 578, + "STOP": 7 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 75, + "INSERT_STATEMENT": 14, + "REMOVE": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 2, + "STOP": 7 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 3, + "INSERT_STATEMENT": 14, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 1, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 74, + "leaf_container_mismatch": 2, + "not_bindable": 504, + "not_canonical_subtree": 576, + "not_container": 15, + "not_removable": 7, + "target_not_canonical": 14 + }, + "visited_total": 1220, + "dead_total": 1192, + "dead_candidate_rate": 0.9770491803278688, + "visited_shares": { + "ADD": 0.000819672131147541, + "BIND_PLACEHOLDER": 0.4745901639344262, + "INSERT_STATEMENT": 0.011475409836065573, + "REMOVE": 0.007377049180327869, + "REMOVE_CONTAINER": 0.012295081967213115, + "REPLACE": 0.002459016393442623, + "REPLACE_STATEMENT": 0.011475409836065573, + "REPLACE_SUBTREE": 0.4737704918032787, + "STOP": 0.005737704918032787 + }, + "verifier_calls": 0, + "states_with_applicable_add": 7, + "states_recalled": 1, + "applicable_add_recall": 0.14285714285714285 + }, + "fixture_04": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 650, + "INSERT_STATEMENT": 16, + "REMOVE": 12, + "REMOVE_CONTAINER": 18, + "REPLACE": 1, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 298, + "STOP": 8 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 74, + "INSERT_STATEMENT": 16, + "REMOVE": 4, + "REPLACE_SUBTREE": 10, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "BIND_PLACEHOLDER": 2, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE_SUBTREE": 5, + "STOP": 7 + }, + "rejection_reasons": { + "duplicate_state": 81, + "leaf_container_mismatch": 1, + "not_bindable": 576, + "not_canonical_subtree": 288, + "not_container": 18, + "not_removable": 8, + "target_not_canonical": 16 + }, + "visited_total": 1020, + "dead_total": 988, + "dead_candidate_rate": 0.9686274509803922, + "visited_shares": { + "ADD": 0.000980392156862745, + "BIND_PLACEHOLDER": 0.6372549019607843, + "INSERT_STATEMENT": 0.01568627450980392, + "REMOVE": 0.011764705882352941, + "REMOVE_CONTAINER": 0.01764705882352941, + "REPLACE": 0.000980392156862745, + "REPLACE_STATEMENT": 0.01568627450980392, + "REPLACE_SUBTREE": 0.29215686274509806, + "STOP": 0.00784313725490196 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 1, + "applicable_add_recall": 0.125 + }, + "fixture_05": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 22, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 17, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2557, + "dead_total": 2513, + "dead_candidate_rate": 0.9827923347673054, + "visited_shares": { + "ADD": 0.0011732499022291747, + "BIND_PLACEHOLDER": 0.3378959718420024, + "INSERT_STATEMENT": 0.008603832616347283, + "INSERT_SUBTREE": 0.0035197497066875244, + "REMOVE": 0.003910833007430583, + "REMOVE_CONTAINER": 0.008603832616347283, + "REPLACE_STATEMENT": 0.006257332811888932, + "REPLACE_SUBTREE": 0.6257332811888933, + "STOP": 0.004301916308173641 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_06": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 22, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 17, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2557, + "dead_total": 2513, + "dead_candidate_rate": 0.9827923347673054, + "visited_shares": { + "ADD": 0.0011732499022291747, + "BIND_PLACEHOLDER": 0.3378959718420024, + "INSERT_STATEMENT": 0.008603832616347283, + "INSERT_SUBTREE": 0.0035197497066875244, + "REMOVE": 0.003910833007430583, + "REMOVE_CONTAINER": 0.008603832616347283, + "REPLACE_STATEMENT": 0.006257332811888932, + "REPLACE_SUBTREE": 0.6257332811888933, + "STOP": 0.004301916308173641 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + }, + "fixture_07": { + "visited": { + "ADD": 3, + "BIND_PLACEHOLDER": 864, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 9, + "REMOVE": 10, + "REMOVE_CONTAINER": 21, + "REPLACE_STATEMENT": 16, + "REPLACE_SUBTREE": 1600, + "STOP": 11 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 288, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 5, + "REPLACE_SUBTREE": 232, + "STOP": 11 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 22, + "INSERT_SUBTREE": 1, + "REMOVE": 2, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 6, + "STOP": 11 + }, + "rejection_reasons": { + "duplicate_state": 519, + "invalid_result": 72, + "not_bindable": 576, + "not_canonical_subtree": 1296, + "not_container": 16, + "not_removable": 8, + "parent_or_comp_precondition": 9, + "target_not_canonical": 16 + }, + "visited_total": 2556, + "dead_total": 2512, + "dead_candidate_rate": 0.9827856025039123, + "visited_shares": { + "ADD": 0.0011737089201877935, + "BIND_PLACEHOLDER": 0.3380281690140845, + "INSERT_STATEMENT": 0.008607198748043818, + "INSERT_SUBTREE": 0.0035211267605633804, + "REMOVE": 0.003912363067292645, + "REMOVE_CONTAINER": 0.008215962441314555, + "REPLACE_STATEMENT": 0.006259780907668232, + "REPLACE_SUBTREE": 0.6259780907668232, + "STOP": 0.004303599374021909 + }, + "verifier_calls": 0, + "states_with_applicable_add": 11, + "states_recalled": 2, + "applicable_add_recall": 0.18181818181818182 + } + } + }, + "calibration": { + "per_action": { + "STOP": { + "target_share": 0.19899244332493704, + "visited_share": 0.005032644178454842, + "gap": -0.1939597991464822 + }, + "REPLACE": { + "target_share": 0.1486146095717884, + "visited_share": 0.000544069640914037, + "gap": -0.14807053993087438 + }, + "ADD": { + "target_share": 0.327455919395466, + "visited_share": 0.001088139281828074, + "gap": -0.32636778011363793 + }, + "REMOVE": { + "target_share": 0.11083123425692695, + "visited_share": 0.005712731229597388, + "gap": -0.10511850302732956 + }, + "ADD_CONTAINER": { + "target_share": 0.017632241813602016, + "visited_share": 0.0, + "gap": -0.017632241813602016 + }, + "REMOVE_CONTAINER": { + "target_share": 0.10327455919395466, + "visited_share": 0.010201305767138194, + "gap": -0.09307325342681647 + }, + "INSERT_SUBTREE": { + "target_share": 0.030226700251889168, + "visited_share": 0.0024483133841131663, + "gap": -0.027778386867776002 + }, + "REPLACE_SUBTREE": { + "target_share": 0.022670025188916875, + "visited_share": 0.5539309031556039, + "gap": 0.531260877966687 + }, + "INSERT_STATEMENT": { + "target_share": 0.0, + "visited_share": 0.010065288356909684, + "gap": 0.010065288356909684 + }, + "REPLACE_STATEMENT": { + "target_share": 0.007556675062972292, + "visited_share": 0.008569096844396083, + "gap": 0.0010124217814237906 + }, + "BIND_PLACEHOLDER": { + "target_share": 0.0327455919395466, + "visited_share": 0.4024075081610446, + "gap": 0.369661916221498 + } + }, + "mean_abs_deviation": 0.16581827351391246 + }, + "dead_candidate_rate": 0.9798694232861807, + "applicable_add_recall": 0.16216216216216217, + "verifier_calls": 14630, + "stop_budget_consumed": 73, + "stop_proposals_visited": 74, + "valid_final_rate": 1.0, + "n_records": 8, + "training_metrics": { + "action": 2.112842082977295, + "stmt": 2.703423023223877, + "comp": 2.7941222190856934, + "slot": 2.7572789192199707, + "value": 0.16673196852207184, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + } + }, + "threshold_checks": { + "t1_add_target_share_gains": { + "cell_b_minus_cell_a": 0.15020724214678874, + "cell_d_minus_cell_c": 0.15020724214678874 + }, + "t1_ok": true, + "t2_valid_final": { + "cell_b_vs_cell_a": true, + "cell_d_vs_cell_c": true + }, + "t2_ok": true, + "t3_stop_accounting": { + "cell_c_vs_cell_a": { + "stop_budget_legacy": 58, + "stop_budget_corrected": 58, + "budget_ok": true, + "valid_final_ok": true + }, + "cell_d_vs_cell_b": { + "stop_budget_legacy": 74, + "stop_budget_corrected": 73, + "budget_ok": true, + "valid_final_ok": true + } + }, + "t3_ok": true, + "t4_lever_isolation": { + "b_minus_a": [ + "corruption" + ], + "c_minus_a": [ + "stop" + ], + "d_minus_b": [ + "stop" + ], + "d_minus_c": [ + "corruption" + ] + }, + "t4_ok": true + }, + "verdicts": { + "corruption_add_balanced": "adopted", + "stop_slot_corrected": "adopted" + }, + "baseline_audit": { + "training_targets": { + "samples": 400, + "n_targets": 378, + "counts": { + "STOP": 71, + "REPLACE": 42, + "ADD": 67, + "REMOVE": 69, + "ADD_CONTAINER": 5, + "REMOVE_CONTAINER": 69, + "INSERT_SUBTREE": 7, + "REPLACE_SUBTREE": 19, + "INSERT_STATEMENT": 0, + "REPLACE_STATEMENT": 7, + "BIND_PLACEHOLDER": 22 + }, + "shares": { + "STOP": 0.18783068783068782, + "REPLACE": 0.1111111111111111, + "ADD": 0.17724867724867724, + "REMOVE": 0.18253968253968253, + "ADD_CONTAINER": 0.013227513227513227, + "REMOVE_CONTAINER": 0.18253968253968253, + "INSERT_SUBTREE": 0.018518518518518517, + "REPLACE_SUBTREE": 0.05026455026455026, + "INSERT_STATEMENT": 0.0, + "REPLACE_STATEMENT": 0.018518518518518517, + "BIND_PLACEHOLDER": 0.0582010582010582 + }, + "inverse_action_weights": null + }, + "decode_demand": { + "overall": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 11092, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + }, + "by_source": { + "fixture_00": { + "visited": { + "BIND_PLACEHOLDER": 576, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 15, + "REPLACE": 3, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 297, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 9, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 185, + "invalid_result": 1, + "leaf_container_mismatch": 1, + "not_bindable": 252, + "not_canonical_subtree": 288, + "not_container": 15, + "not_leaf_component": 144, + "not_removable": 6, + "target_not_canonical": 2 + }, + "visited_total": 918, + "dead_total": 894, + "dead_candidate_rate": 0.9738562091503268, + "visited_shares": { + "BIND_PLACEHOLDER": 0.6274509803921569, + "INSERT_STATEMENT": 0.013071895424836602, + "REMOVE": 0.007625272331154684, + "REMOVE_CONTAINER": 0.016339869281045753, + "REPLACE": 0.0032679738562091504, + "REPLACE_STATEMENT": 0.002178649237472767, + "REPLACE_SUBTREE": 0.3235294117647059, + "STOP": 0.006535947712418301 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_01": { + "visited": { + "ADD": 2, + "BIND_PLACEHOLDER": 648, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 17, + "REMOVE": 3, + "REMOVE_CONTAINER": 14, + "REPLACE": 2, + "REPLACE_STATEMENT": 2, + "REPLACE_SUBTREE": 1022, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 180, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 3, + "REPLACE_SUBTREE": 230, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 17, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 1, + "REPLACE_SUBTREE": 7, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 407, + "invalid_result": 73, + "leaf_container_mismatch": 1, + "not_bindable": 432, + "not_canonical_subtree": 720, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 2, + "parent_or_comp_precondition": 17, + "target_not_canonical": 2 + }, + "visited_total": 1737, + "dead_total": 1701, + "dead_candidate_rate": 0.9792746113989638, + "visited_shares": { + "ADD": 0.0011514104778353484, + "BIND_PLACEHOLDER": 0.37305699481865284, + "INSERT_STATEMENT": 0.010362694300518135, + "INSERT_SUBTREE": 0.00978698906160046, + "REMOVE": 0.0017271157167530224, + "REMOVE_CONTAINER": 0.008059873344847437, + "REPLACE": 0.0011514104778353484, + "REPLACE_STATEMENT": 0.0011514104778353484, + "REPLACE_SUBTREE": 0.588370754173863, + "STOP": 0.0051813471502590676 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_02": { + "visited": { + "BIND_PLACEHOLDER": 504, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1431, + "dead_total": 1407, + "dead_candidate_rate": 0.9832285115303984, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3522012578616352, + "INSERT_STATEMENT": 0.008385744234800839, + "REMOVE": 0.004891684136967156, + "REMOVE_CONTAINER": 0.007686932215234102, + "REPLACE": 0.011879804332634521, + "REPLACE_STATEMENT": 0.0041928721174004195, + "REPLACE_SUBTREE": 0.6065688329839273, + "STOP": 0.0041928721174004195 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_03": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 8, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 72, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 75, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 72, + "not_removable": 6, + "target_not_canonical": 8 + }, + "visited_total": 1397, + "dead_total": 1373, + "dead_candidate_rate": 0.9828203292770222, + "visited_shares": { + "BIND_PLACEHOLDER": 0.3350035790980673, + "INSERT_STATEMENT": 0.008589835361488905, + "REMOVE": 0.005010737294201861, + "REMOVE_CONTAINER": 0.007874015748031496, + "REPLACE": 0.012168933428775949, + "REPLACE_STATEMENT": 0.00572655690765927, + "REPLACE_SUBTREE": 0.6213314244810307, + "STOP": 0.004294917680744453 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_04": { + "visited": { + "BIND_PLACEHOLDER": 468, + "INSERT_STATEMENT": 12, + "REMOVE": 7, + "REMOVE_CONTAINER": 11, + "REPLACE": 17, + "REPLACE_STATEMENT": 6, + "REPLACE_SUBTREE": 868, + "STOP": 6 + }, + "applicable": { + "BIND_PLACEHOLDER": 108, + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 4, + "REPLACE_SUBTREE": 4, + "STOP": 6 + }, + "selected": { + "INSERT_STATEMENT": 12, + "REMOVE": 1, + "REPLACE": 3, + "REPLACE_SUBTREE": 2, + "STOP": 6 + }, + "rejection_reasons": { + "duplicate_state": 111, + "invalid_result": 6, + "leaf_container_mismatch": 5, + "no_op": 2, + "not_bindable": 324, + "not_canonical_subtree": 864, + "not_container": 11, + "not_leaf_component": 36, + "not_removable": 6, + "target_not_canonical": 6 + }, + "visited_total": 1395, + "dead_total": 1371, + "dead_candidate_rate": 0.9827956989247312, + "visited_shares": { + "BIND_PLACEHOLDER": 0.33548387096774196, + "INSERT_STATEMENT": 0.008602150537634409, + "REMOVE": 0.005017921146953405, + "REMOVE_CONTAINER": 0.007885304659498209, + "REPLACE": 0.012186379928315413, + "REPLACE_STATEMENT": 0.004301075268817204, + "REPLACE_SUBTREE": 0.6222222222222222, + "STOP": 0.004301075268817204 + }, + "verifier_calls": 0, + "states_with_applicable_add": 6, + "states_recalled": 0, + "applicable_add_recall": 0.0 + }, + "fixture_05": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 48, + "REMOVE": 4, + "REMOVE_CONTAINER": 9, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 9, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 50, + "target_not_canonical": 4 + }, + "visited_total": 1226, + "dead_total": 1194, + "dead_candidate_rate": 0.9738988580750407, + "visited_shares": { + "ADD": 0.0032626427406199023, + "BIND_PLACEHOLDER": 0.44045676998368677, + "INSERT_STATEMENT": 0.013050570962479609, + "INSERT_SUBTREE": 0.03915171288743882, + "REMOVE": 0.0032626427406199023, + "REMOVE_CONTAINER": 0.00734094616639478, + "REPLACE": 0.004893964110929853, + "REPLACE_STATEMENT": 0.0032626427406199023, + "REPLACE_SUBTREE": 0.47879282218597063, + "STOP": 0.0065252854812398045 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + }, + "fixture_06": { + "visited": { + "ADD": 1, + "BIND_PLACEHOLDER": 720, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 33, + "REMOVE": 4, + "REMOVE_CONTAINER": 14, + "REPLACE": 5, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 1019, + "STOP": 9 + }, + "applicable": { + "ADD": 1, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 9 + }, + "selected": { + "INSERT_STATEMENT": 18, + "INSERT_SUBTREE": 1, + "REMOVE": 1, + "REMOVE_CONTAINER": 2, + "REPLACE": 1, + "REPLACE_SUBTREE": 4, + "STOP": 9 + }, + "rejection_reasons": { + "duplicate_state": 332, + "invalid_result": 38, + "leaf_container_mismatch": 2, + "not_bindable": 396, + "not_canonical_subtree": 864, + "not_container": 12, + "not_leaf_component": 108, + "not_removable": 3, + "parent_or_comp_precondition": 32, + "target_not_canonical": 4 + }, + "visited_total": 1827, + "dead_total": 1791, + "dead_candidate_rate": 0.9802955665024631, + "visited_shares": { + "ADD": 0.0005473453749315818, + "BIND_PLACEHOLDER": 0.39408866995073893, + "INSERT_STATEMENT": 0.009852216748768473, + "INSERT_SUBTREE": 0.0180623973727422, + "REMOVE": 0.0021893814997263274, + "REMOVE_CONTAINER": 0.007662835249042145, + "REPLACE": 0.002736726874657909, + "REPLACE_STATEMENT": 0.0021893814997263274, + "REPLACE_SUBTREE": 0.5577449370552818, + "STOP": 0.0049261083743842365 + }, + "verifier_calls": 0, + "states_with_applicable_add": 9, + "states_recalled": 1, + "applicable_add_recall": 0.1111111111111111 + }, + "fixture_07": { + "visited": { + "ADD": 4, + "BIND_PLACEHOLDER": 540, + "INSERT_STATEMENT": 16, + "INSERT_SUBTREE": 40, + "REMOVE": 4, + "REMOVE_CONTAINER": 10, + "REPLACE": 6, + "REPLACE_STATEMENT": 4, + "REPLACE_SUBTREE": 587, + "STOP": 8 + }, + "applicable": { + "ADD": 2, + "BIND_PLACEHOLDER": 216, + "INSERT_STATEMENT": 16, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 119, + "STOP": 8 + }, + "selected": { + "ADD": 1, + "INSERT_STATEMENT": 15, + "REMOVE": 1, + "REPLACE": 1, + "REPLACE_SUBTREE": 6, + "STOP": 8 + }, + "rejection_reasons": { + "duplicate_state": 331, + "invalid_result": 38, + "leaf_container_mismatch": 3, + "not_bindable": 288, + "not_canonical_subtree": 432, + "not_container": 10, + "not_leaf_component": 36, + "not_removable": 3, + "parent_or_comp_precondition": 42, + "target_not_canonical": 4 + }, + "visited_total": 1219, + "dead_total": 1187, + "dead_candidate_rate": 0.9737489745693191, + "visited_shares": { + "ADD": 0.003281378178835111, + "BIND_PLACEHOLDER": 0.44298605414273995, + "INSERT_STATEMENT": 0.013125512715340444, + "INSERT_SUBTREE": 0.03281378178835111, + "REMOVE": 0.003281378178835111, + "REMOVE_CONTAINER": 0.008203445447087777, + "REPLACE": 0.004922067268252666, + "REPLACE_STATEMENT": 0.003281378178835111, + "REPLACE_SUBTREE": 0.4815422477440525, + "STOP": 0.006562756357670222 + }, + "verifier_calls": 0, + "states_with_applicable_add": 8, + "states_recalled": 2, + "applicable_add_recall": 0.25 + } + }, + "by_suite": { + "train": { + "visited": { + "ADD": 11, + "BIND_PLACEHOLDER": 4464, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 138, + "REMOVE": 43, + "REMOVE_CONTAINER": 95, + "REPLACE": 73, + "REPLACE_STATEMENT": 36, + "REPLACE_SUBTREE": 6116, + "STOP": 58 + }, + "applicable": { + "ADD": 6, + "BIND_PLACEHOLDER": 1296, + "INSERT_STATEMENT": 116, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 5, + "REPLACE": 16, + "REPLACE_SUBTREE": 608, + "STOP": 58 + }, + "selected": { + "ADD": 2, + "INSERT_STATEMENT": 113, + "INSERT_SUBTREE": 2, + "REMOVE": 8, + "REMOVE_CONTAINER": 3, + "REPLACE": 13, + "REPLACE_SUBTREE": 33, + "STOP": 58 + }, + "rejection_reasons": { + "duplicate_state": 1883, + "invalid_result": 206, + "leaf_container_mismatch": 25, + "no_op": 6, + "not_bindable": 2628, + "not_canonical_subtree": 5328, + "not_container": 90, + "not_leaf_component": 540, + "not_removable": 35, + "parent_or_comp_precondition": 141, + "target_not_canonical": 36 + }, + "visited_total": 11150, + "dead_total": 10918, + "dead_candidate_rate": 0.9791928251121076, + "visited_shares": { + "ADD": 0.0009865470852017937, + "BIND_PLACEHOLDER": 0.4003587443946188, + "INSERT_STATEMENT": 0.010403587443946188, + "INSERT_SUBTREE": 0.012376681614349776, + "REMOVE": 0.0038565022421524664, + "REMOVE_CONTAINER": 0.008520179372197309, + "REPLACE": 0.006547085201793722, + "REPLACE_STATEMENT": 0.003228699551569507, + "REPLACE_SUBTREE": 0.5485201793721973, + "STOP": 0.005201793721973094 + }, + "verifier_calls": 0, + "states_with_applicable_add": 58, + "states_recalled": 6, + "applicable_add_recall": 0.10344827586206896 + } + } + }, + "calibration": { + "per_action": { + "STOP": { + "target_share": 0.18783068783068782, + "visited_share": 0.005201793721973094, + "gap": -0.18262889410871472 + }, + "REPLACE": { + "target_share": 0.1111111111111111, + "visited_share": 0.006547085201793722, + "gap": -0.10456402590931738 + }, + "ADD": { + "target_share": 0.17724867724867724, + "visited_share": 0.0009865470852017937, + "gap": -0.17626213016347544 + }, + "REMOVE": { + "target_share": 0.18253968253968253, + "visited_share": 0.0038565022421524664, + "gap": -0.17868318029753005 + }, + "ADD_CONTAINER": { + "target_share": 0.013227513227513227, + "visited_share": 0.0, + "gap": -0.013227513227513227 + }, + "REMOVE_CONTAINER": { + "target_share": 0.18253968253968253, + "visited_share": 0.008520179372197309, + "gap": -0.1740195031674852 + }, + "INSERT_SUBTREE": { + "target_share": 0.018518518518518517, + "visited_share": 0.012376681614349776, + "gap": -0.006141836904168741 + }, + "REPLACE_SUBTREE": { + "target_share": 0.05026455026455026, + "visited_share": 0.5485201793721973, + "gap": 0.4982556291076471 + }, + "INSERT_STATEMENT": { + "target_share": 0.0, + "visited_share": 0.010403587443946188, + "gap": 0.010403587443946188 + }, + "REPLACE_STATEMENT": { + "target_share": 0.018518518518518517, + "visited_share": 0.003228699551569507, + "gap": -0.01528981896694901 + }, + "BIND_PLACEHOLDER": { + "target_share": 0.0582010582010582, + "visited_share": 0.4003587443946188, + "gap": 0.34215768619356063 + } + }, + "mean_abs_deviation": 0.1546939823173007 + }, + "loss_reweighting_prediction": { + "predicted": true, + "rule": "predict reweighting iff applicable-ADD recall < 0.5 or |ADD visited share - ADD target share| > 0.2", + "add_share_gap": -0.17626213016347544, + "applicable_add_recall": 0.10344827586206896 + } + }, + "loss_decision": { + "audit_prediction": { + "predicted": true, + "rule": "predict reweighting iff applicable-ADD recall < 0.5 or |ADD visited share - ADD target share| > 0.2", + "add_share_gap": -0.17626213016347544, + "applicable_add_recall": 0.10344827586206896 + }, + "decision": "not_added_preregistered_cells_unchanged", + "rationale": "The audit's preregistered rule predicted reweighting=True. The matched 2x2 was preregistered without a loss arm; adding one post-hoc would break lever isolation, so it is deferred to a follow-up preregistered cell rather than combined here." + }, + "honesty": "Fixture-scale matched cells; decode-demand metrics come from SLM-310 per-proposal telemetry (visited = enumerated candidates the decode loop actually considered). A fixture verdict is wiring/distribution evidence, not a production ship claim. Negative results are retained per lever, never combined.", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "ae5448c594233b304ea22720604a962ad041fe63", + "code_dirty": true, + "components": { + "harness.experiments.slm310_action_alignment": "v1", + "harness.experiments.slm299_edit_reachability": "v4" + }, + "stamped_at": "2026-07-24T22:07:47.206210+00:00" + } +} diff --git a/docs/design/iter-slm310-action-alignment-20260724.md b/docs/design/iter-slm310-action-alignment-20260724.md new file mode 100644 index 000000000..7ab2a51f6 --- /dev/null +++ b/docs/design/iter-slm310-action-alignment-20260724.md @@ -0,0 +1,38 @@ +# SLM-310 (LAR2-03): action alignment — corruption sampler × STOP-slot accounting + +**Verdicts: corruption=ADD-balanced `adopted`, STOP-slot corrected `adopted`** (fixture-scale matched cells; not a ship claim) + +## Preregistered thresholds (locked before results) + +- T1: ADD training-target share gain >= 0.1 (both STOP arms) +- T2: ADD-balanced never reduces valid-final rate +- T3: corrected STOP accounting consumes <= legacy STOP budget and never reduces valid-final rate +- T4: lever isolation is structural (single-lever config diffs) +- rule: per lever: adopted iff its thresholds hold in both cells of the matched pair, else rejected; levers are never combined + +## Cells (levers isolated) + +| cell | corruption | STOP accounting | dead-candidate rate | applicable-ADD recall | ADD target share | verifier calls | STOP budget | valid-final | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| cell_a | default | legacy | 0.9792 | 0.1034 | 0.1772 | 11092 | 58 | 1.0000 | +| cell_b | ADD-balanced | legacy | 0.9799 | 0.1622 | 0.3275 | 14629 | 74 | 1.0000 | +| cell_c | default | corrected | 0.9792 | 0.1034 | 0.1772 | 11092 | 58 | 1.0000 | +| cell_d | ADD-balanced | corrected | 0.9799 | 0.1622 | 0.3275 | 14630 | 73 | 1.0000 | + +## Threshold checks + +- T1 gains: {'cell_b_minus_cell_a': 0.15020724214678874, 'cell_d_minus_cell_c': 0.15020724214678874} → ok=True +- T2 valid-final: {'cell_b_vs_cell_a': True, 'cell_d_vs_cell_c': True} → ok=True +- T3 STOP accounting: {'cell_c_vs_cell_a': {'stop_budget_legacy': 58, 'stop_budget_corrected': 58, 'budget_ok': True, 'valid_final_ok': True}, 'cell_d_vs_cell_b': {'stop_budget_legacy': 74, 'stop_budget_corrected': 73, 'budget_ok': True, 'valid_final_ok': True}} → ok=True +- T4 isolation: {'b_minus_a': ['corruption'], 'c_minus_a': ['stop'], 'd_minus_b': ['stop'], 'd_minus_c': ['corruption']} → ok=True + +## Baseline distribution audit (cell A) + +- dead-candidate rate 0.9792, applicable-ADD recall 0.1034, calibration MAD 0.1547 +- top rejection reasons: {'duplicate_state': 1883, 'invalid_result': 206, 'leaf_container_mismatch': 25, 'no_op': 6, 'not_bindable': 2628, 'not_canonical_subtree': 5328, 'not_container': 90, 'not_leaf_component': 540, 'not_removable': 35, 'parent_or_comp_precondition': 141, 'target_not_canonical': 36} +- loss-reweighting prediction: True (predict reweighting iff applicable-ADD recall < 0.5 or |ADD visited share - ADD target share| > 0.2) +- decision: not_added_preregistered_cells_unchanged — The audit's preregistered rule predicted reweighting=True. The matched 2x2 was preregistered without a loss arm; adding one post-hoc would break lever isolation, so it is deferred to a follow-up preregistered cell rather than combined here. + +## Honesty + +Fixture-scale matched cells; decode-demand metrics come from SLM-310 per-proposal telemetry (visited = enumerated candidates the decode loop actually considered). A fixture verdict is wiring/distribution evidence, not a production ship claim. Negative results are retained per lever, never combined. diff --git a/scripts/run_slm310_action_alignment.py b/scripts/run_slm310_action_alignment.py new file mode 100644 index 000000000..bb5257f4d --- /dev/null +++ b/scripts/run_slm310_action_alignment.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +"""SLM-310 (LAR2-03): matched fixture experiment — action alignment in X22. + +Question: does aligning the corruption sampler with a declared inverse-action +distribution (ADD-balanced) close the training-target vs decode-demand gap, +and does corrected STOP-slot accounting (STOP consumes an expand_per_state +slot only when its frozen candidate is retained) change search efficiency — +each lever isolated, at equal model / data / optimizer budget? + +Cells (2×2, levers isolated; A is the baseline): + +- A: default corruption sampler × legacy STOP accounting +- B: ADD-balanced corruption sampler × legacy STOP accounting +- C: default corruption sampler × corrected STOP accounting +- D: ADD-balanced corruption sampler × corrected STOP accounting + +Preregistered (written into the output payload BEFORE any result; locked — +do not edit after outcomes are visible): + +- T1 (corruption lever): within each STOP arm, the ADD-balanced cell must + raise the ADD training-target share by >= 0.10 absolute vs the default + cell (sampler hits its declared distribution); +- T2 (corruption lever): within each STOP arm, the ADD-balanced cell must + not reduce the valid-final rate below the default cell's (no semantic + regression); +- T3 (STOP lever): within each corruption arm, the corrected cell must + consume strictly fewer or equal STOP budget slots than the legacy cell + (duplicate STOPs no longer burn budget) and must not reduce the + valid-final rate; +- T4 (isolation): B−A differs only in the sampler, C−A only in STOP + accounting — verified structurally by config diff, not by narration. +- Verdict per lever: ``adopted`` iff its thresholds hold in BOTH cells of + the pair, else ``rejected``. A negative result is retained, never + combined. + +The distribution audit (training-target vs decode-demand) runs on the +baseline cell and its preregistered prediction decides whether a +class-balanced/focal action loss is warranted; the prediction and the +decision are recorded in the payload (no loss arm was added unless the audit +predicted it — see ``loss_decision``). + +Writes ``docs/design/iter-slm310-action-alignment-20260724.{json,md}``. + +Example: + python -m scripts.run_slm310_action_alignment --steps 8 +""" + +from __future__ import annotations + +import argparse +import json +import random +from pathlib import Path + +import torch + +from slm_training.dsl.parser import validate +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm310_action_alignment import ( + action_calibration, + decode_demand_distribution, + run_distribution_audit, + training_target_distribution, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_STOP, + TreeEditDiffusionConfig, + TreeEditDiffusionModel, +) +from slm_training.versioning import build_version_stamp + +EXPERIMENT_ID = "slm310-action-alignment" +DEFAULT_JSON_OUT = Path("docs/design/iter-slm310-action-alignment-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm310-action-alignment-20260724.md") + +# PREREGISTERED thresholds (locked before any run; deviations are append-only +# and exploratory). +PREREGISTERED_THRESHOLDS = { + "t1_add_target_share_gain_min": 0.10, + "t2_valid_final_no_regression": True, + "t3_stop_budget_not_worse_and_valid_final_no_regression": True, + "t4_lever_isolation_structural": True, + "verdict_rule": ( + "per lever: adopted iff its thresholds hold in both cells of the " + "matched pair, else rejected; levers are never combined" + ), +} + +# Declared inverse-action distribution for the ADD-balanced corruption arm: +# ADD lifted well above its default ~10% share (only REMOVE mutations yield +# ADD inverses), all other reachable inverse actions uniform. +ADD_BALANCED_DISTRIBUTION = { + "REPLACE": 1.0, + "ADD": 4.0, + "REMOVE": 1.0, + "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, + "INSERT_SUBTREE": 1.0, + "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, + "BIND_PLACEHOLDER": 1.0, +} + +CELLS = { + "cell_a": {"corruption": None, "stop": "legacy"}, + "cell_b": {"corruption": ADD_BALANCED_DISTRIBUTION, "stop": "legacy"}, + "cell_c": {"corruption": None, "stop": "corrected"}, + "cell_d": {"corruption": ADD_BALANCED_DISTRIBUTION, "stop": "corrected"}, +} + +# Tiny fixture corpus (same shape as the SLM-308 matched experiment). +FIXTURE_PROGRAMS: list[tuple[str, str, list[str]]] = [ + ("hero with cta", 'root = Stack([t, b], "column")\nt = TextContent(":hero.title")\nb = Button(":cta.label")', [":hero.title", ":cta.label"]), + ("simple text", 'root = Stack([t], "column")\nt = TextContent(":body")', [":body"]), + ("form with input", 'root = Form([i, s], "column")\ni = TextInput(":form.email")\ns = Button(":form.submit")', [":form.email", ":form.submit"]), + ("card with image", 'root = Stack([c], "column")\nc = Card([im, t], "column")\nim = Image(":img.src")\nt = TextContent(":img.caption")', [":img.src", ":img.caption"]), + ("two texts", 'root = Stack([a, b], "column")\na = TextContent(":title")\nb = TextContent(":subtitle")', [":title", ":subtitle"]), + ("button only", 'root = Stack([b], "column")\nb = Button(":cta")', [":cta"]), + ("nested card", 'root = Stack([c], "column")\nc = Card([t], "column")\nt = TextContent(":card.body")', [":card.body"]), + ("form pair", 'root = Form([i], "column")\ni = TextInput(":q")', [":q"]), +] + + +def build_fixture_records() -> list[ExampleRecord]: + return [ + ExampleRecord( + id=f"fixture_{i:02d}", + prompt=prompt, + openui=openui, + placeholders=list(placeholders), + split="train", + ) + for i, (prompt, openui, placeholders) in enumerate(FIXTURE_PROGRAMS) + ] + + +def train_cell( + records: list[ExampleRecord], + corruption: dict[str, float] | None, + stop: str, + *, + steps: int, + batch_size: int, + seed: int, +) -> TreeEditDiffusionModel: + """Train one cell; cells share corpus, init seed, batch order, optimizer + budget, and value_label_mode — only the cell's lever(s) differ.""" + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + d_model=32, + n_heads=4, + context_layers=1, + denoiser_layers=1, + seed=seed, + max_chain=3, + value_label_mode="mutation_count", + context_backend="scratch", + corruption_action_distribution=corruption, + stop_slot_accounting=stop, + ) + model = TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=3e-3) + order = random.Random(0) # identical batch order across cells + model.train() + for _ in range(steps): + batch = [records[order.randrange(len(records))] for _ in range(batch_size)] + loss = model.training_loss(batch) + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +def score_cell( + model: TreeEditDiffusionModel, + records: list[ExampleRecord], + *, + audit_samples: int, + seed: int, +) -> dict: + """Per-cell metrics: dead-candidate rate, applicable-ADD recall, action + calibration, verifier calls, semantic outcomes (valid-final rate).""" + weights = model._inverse_action_weights() + training = training_target_distribution( + records, + model.space, + seed=seed, + samples=audit_samples, + max_chain=model.config.max_chain, + inverse_action_weights=weights, + ) + demand = decode_demand_distribution(model, records, include_outputs=True) + outputs = demand.pop("outputs") + valid_final = 0 + for text in outputs: + try: + validate(text) + valid_final += 1 + except Exception: # noqa: BLE001 + pass + calib = action_calibration(training, demand["overall"]) + overall = demand["overall"] + proposals = [ + p for ev in demand["evidence"] for p in ev.get("proposals", []) + ] + stop_budget = sum( + 1 for p in proposals if p["action"] == ACTION_STOP and p["consumed_budget"] + ) + stop_visited = sum(1 for p in proposals if p["action"] == ACTION_STOP) + return { + "training_targets": training, + "decode_demand": { + "overall": overall, + "by_suite": demand["by_suite"], + "by_source": demand["by_source"], + }, + "calibration": calib, + "dead_candidate_rate": overall["dead_candidate_rate"], + "applicable_add_recall": overall["applicable_add_recall"], + "verifier_calls": overall["verifier_calls"], + "stop_budget_consumed": stop_budget, + "stop_proposals_visited": stop_visited, + "valid_final_rate": valid_final / max(len(records), 1), + "n_records": len(records), + "training_metrics": model.last_training_metrics, + } + + +def build_report( + records: list[ExampleRecord], + *, + steps: int, + batch_size: int, + seed: int, + audit_samples: int, +) -> dict: + cells: dict[str, dict] = {} + for cell, spec in CELLS.items(): + model = train_cell( + records, + spec["corruption"], + spec["stop"], + steps=steps, + batch_size=batch_size, + seed=seed, + ) + metrics = score_cell(model, records, audit_samples=audit_samples, seed=seed) + cells[cell] = {"levers": spec, "metrics": metrics} + + # T4: lever isolation is structural — pairwise config diffs must name + # exactly one lever. + def lever_diff(x: str, y: str) -> list[str]: + return [ + key + for key in ("corruption", "stop") + if CELLS[x][key] != CELLS[y][key] + ] + + isolation = { + "b_minus_a": lever_diff("cell_b", "cell_a"), + "c_minus_a": lever_diff("cell_c", "cell_a"), + "d_minus_b": lever_diff("cell_d", "cell_b"), + "d_minus_c": lever_diff("cell_d", "cell_c"), + } + t4_ok = ( + isolation["b_minus_a"] == ["corruption"] + and isolation["c_minus_a"] == ["stop"] + and isolation["d_minus_b"] == ["stop"] + and isolation["d_minus_c"] == ["corruption"] + ) + + thr = PREREGISTERED_THRESHOLDS + checks: dict[str, dict] = {} + # T1/T2: corruption lever, compared within each STOP arm (A→B, C→D). + t1 = {} + t2 = {} + for base, balanced in (("cell_a", "cell_b"), ("cell_c", "cell_d")): + a_add = cells[base]["metrics"]["training_targets"]["shares"]["ADD"] + b_add = cells[balanced]["metrics"]["training_targets"]["shares"]["ADD"] + t1[f"{balanced}_minus_{base}"] = b_add - a_add + a_vf = cells[base]["metrics"]["valid_final_rate"] + b_vf = cells[balanced]["metrics"]["valid_final_rate"] + t2[f"{balanced}_vs_{base}"] = b_vf >= a_vf + t1_ok = all(g >= thr["t1_add_target_share_gain_min"] for g in t1.values()) + t2_ok = all(t2.values()) + # T3: STOP lever, compared within each corruption arm (A→C, B→D). + t3 = {} + for legacy, corrected in (("cell_a", "cell_c"), ("cell_b", "cell_d")): + leg = cells[legacy]["metrics"] + cor = cells[corrected]["metrics"] + t3[f"{corrected}_vs_{legacy}"] = { + "stop_budget_legacy": leg["stop_budget_consumed"], + "stop_budget_corrected": cor["stop_budget_consumed"], + "budget_ok": cor["stop_budget_consumed"] <= leg["stop_budget_consumed"], + "valid_final_ok": cor["valid_final_rate"] >= leg["valid_final_rate"], + } + t3_ok = all(v["budget_ok"] and v["valid_final_ok"] for v in t3.values()) + + checks = { + "t1_add_target_share_gains": t1, + "t1_ok": t1_ok, + "t2_valid_final": t2, + "t2_ok": t2_ok, + "t3_stop_accounting": t3, + "t3_ok": t3_ok, + "t4_lever_isolation": isolation, + "t4_ok": t4_ok, + } + verdicts = { + "corruption_add_balanced": "adopted" if (t1_ok and t2_ok) else "rejected", + "stop_slot_corrected": "adopted" if t3_ok else "rejected", + } + + # Distribution audit on the baseline cell decides whether a + # class-balanced/focal action loss is warranted (preregistered rule). + audit_model = train_cell( + records, None, "legacy", steps=steps, batch_size=batch_size, seed=seed + ) + audit = run_distribution_audit( + audit_model, records, seed=seed, samples=audit_samples, max_chain=3 + ) + predicted = audit["loss_reweighting_prediction"]["predicted"] + loss_decision = { + "audit_prediction": audit["loss_reweighting_prediction"], + "decision": ( + "not_added" + if not predicted + else "not_added_preregistered_cells_unchanged" + ), + "rationale": ( + "The audit's preregistered rule predicted " + f"reweighting={predicted}. The matched 2x2 was preregistered " + "without a loss arm; adding one post-hoc would break lever " + "isolation, so it is deferred to a follow-up preregistered cell " + "rather than combined here." + ), + } + + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-310", + "question": ( + "Does ADD-balanced corruption sampling close the train/demand " + "action gap, and does corrected STOP-slot accounting improve " + "search budget use — each lever isolated at fixture scale?" + ), + "preregistered_thresholds": PREREGISTERED_THRESHOLDS, + "config": { + "steps": steps, + "batch_size": batch_size, + "seed": seed, + "audit_samples": audit_samples, + "n_records": len(records), + "value_label_mode": "mutation_count", + "add_balanced_distribution": ADD_BALANCED_DISTRIBUTION, + "cells": CELLS, + }, + "cells": cells, + "threshold_checks": checks, + "verdicts": verdicts, + "baseline_audit": audit, + "loss_decision": loss_decision, + "honesty": ( + "Fixture-scale matched cells; decode-demand metrics come from " + "SLM-310 per-proposal telemetry (visited = enumerated candidates " + "the decode loop actually considered). A fixture verdict is " + "wiring/distribution evidence, not a production ship claim. " + "Negative results are retained per lever, never combined." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm310_action_alignment", + "harness.experiments.slm299_edit_reachability", + ) + return payload + + +def render_markdown(payload: dict) -> str: + cells = payload["cells"] + thr = payload["preregistered_thresholds"] + checks = payload["threshold_checks"] + + def fmt(x: object) -> str: + return "n/a" if x is None else (f"{x:.4f}" if isinstance(x, float) else str(x)) + + lines = [ + "# SLM-310 (LAR2-03): action alignment — corruption sampler × STOP-slot accounting", + "", + f"**Verdicts: corruption=ADD-balanced `{payload['verdicts']['corruption_add_balanced']}`, " + f"STOP-slot corrected `{payload['verdicts']['stop_slot_corrected']}`** " + "(fixture-scale matched cells; not a ship claim)", + "", + "## Preregistered thresholds (locked before results)", + "", + f"- T1: ADD training-target share gain >= {thr['t1_add_target_share_gain_min']} (both STOP arms)", + "- T2: ADD-balanced never reduces valid-final rate", + "- T3: corrected STOP accounting consumes <= legacy STOP budget and never reduces valid-final rate", + "- T4: lever isolation is structural (single-lever config diffs)", + f"- rule: {thr['verdict_rule']}", + "", + "## Cells (levers isolated)", + "", + "| cell | corruption | STOP accounting | dead-candidate rate | applicable-ADD recall | ADD target share | verifier calls | STOP budget | valid-final |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for cell, entry in cells.items(): + m = entry["metrics"] + lev = entry["levers"] + lines.append( + f"| {cell} | {'ADD-balanced' if lev['corruption'] else 'default'} " + f"| {lev['stop']} | {fmt(m['dead_candidate_rate'])} " + f"| {fmt(m['applicable_add_recall'])} " + f"| {fmt(m['training_targets']['shares']['ADD'])} " + f"| {m['verifier_calls']} | {m['stop_budget_consumed']} " + f"| {fmt(m['valid_final_rate'])} |" + ) + lines += [ + "", + "## Threshold checks", + "", + f"- T1 gains: {checks['t1_add_target_share_gains']} → ok={checks['t1_ok']}", + f"- T2 valid-final: {checks['t2_valid_final']} → ok={checks['t2_ok']}", + f"- T3 STOP accounting: {checks['t3_stop_accounting']} → ok={checks['t3_ok']}", + f"- T4 isolation: {checks['t4_lever_isolation']} → ok={checks['t4_ok']}", + "", + "## Baseline distribution audit (cell A)", + "", + ] + audit = payload["baseline_audit"] + lines.append( + f"- dead-candidate rate {fmt(audit['decode_demand']['overall']['dead_candidate_rate'])}, " + f"applicable-ADD recall {fmt(audit['decode_demand']['overall']['applicable_add_recall'])}, " + f"calibration MAD {fmt(audit['calibration']['mean_abs_deviation'])}" + ) + pred = payload["loss_decision"]["audit_prediction"] + lines += [ + f"- top rejection reasons: {audit['decode_demand']['overall']['rejection_reasons']}", + f"- loss-reweighting prediction: {pred['predicted']} ({pred['rule']})", + f"- decision: {payload['loss_decision']['decision']} — {payload['loss_decision']['rationale']}", + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=8) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--audit-samples", type=int, default=400) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + args = parser.parse_args(argv) + + records = build_fixture_records() + payload = build_report( + records, + steps=args.steps, + batch_size=args.batch_size, + seed=args.seed, + audit_samples=args.audit_samples, + ) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"verdicts={payload['verdicts']} " + f"t1={payload['threshold_checks']['t1_ok']} " + f"t3={payload['threshold_checks']['t3_ok']} " + f"-> {args.json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm310_action_alignment.py b/src/slm_training/harnesses/experiments/slm310_action_alignment.py new file mode 100644 index 000000000..8e9bddbca --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm310_action_alignment.py @@ -0,0 +1,330 @@ +"""SLM-310 (LAR2-03): action-alignment audit for the X22 tree-edit model. + +Question: does the inverse-edit supervision distribution (what training +targets ask the policy to repair) match the decode-demand distribution (what +the value-guided beam actually proposes, finds applicable, and selects)? + +Two measurement sides, both fixture-scale and deterministic: + +- ``training_target_distribution`` replays the training-time sampling loop + (a ``stop_fraction`` share of clean-state STOP targets + 1..max_chain + forward-noise chains whose last inverse edit is the supervised target) and + counts inverse actions. +- ``decode_demand_distribution`` decodes every record and aggregates the + SLM-310 per-proposal telemetry (``evidence["proposals"]`` / + ``evidence["states"]``): per action, how many enumerated candidates were + visited, applicable, and selected — overall, by source record, by suite. + +Derived metrics: + +- dead-candidate rate: visited proposals with a rejection reason + (inapplicable or duplicate) over all visited proposals; +- applicable-ADD recall: over the exact decode states where at least one + ADD edit is applicable in the full edit space, the fraction where the + decode loop visited at least one applicable ADD proposal; +- action calibration: per-action decode-visited share vs training-target + share (mean absolute deviation over actions). +""" + +from __future__ import annotations + +import random +from collections import Counter +from typing import Any + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.model_build.plugin import GenerationRequest +from slm_training.models.tree_edit_diffusion import ( + ACTION_ADD, + ACTION_NAMES, + ACTION_STOP, + Edit, + LEAF_COMPONENTS, + MAX_SLOTS, + TreeEditDiffusionModel, + TreeEditSpace, + parse_statements, +) + +#: Preregistered audit rule (locked before any experiment run): recommend a +#: class-balanced/focal action loss only when the measured misalignment is +#: large enough — applicable-ADD recall below this floor OR an ADD +#: train/demand share gap above this bound. +ADD_RECALL_FLOOR = 0.5 +ADD_SHARE_GAP = 0.20 + + +def _inventory(record: ExampleRecord) -> list[str]: + return [ + p if p.startswith(":") else f":{p}" + for p in (record.placeholders or []) + ][:MAX_SLOTS] + + +def training_target_distribution( + records: list[ExampleRecord], + space: TreeEditSpace, + *, + seed: int = 0, + samples: int = 400, + max_chain: int = 4, + inverse_action_weights: dict[int, float] | None = None, + stop_fraction: float = 0.2, +) -> dict[str, Any]: + """Replay the training-time target sampler and count inverse actions. + + Mirrors ``TreeEditDiffusionModel.training_loss``: with probability + ``stop_fraction`` the target is STOP (clean state); otherwise a + 1..max_chain forward-noise chain is applied and the last inverse edit is + the target. Records whose program fails to parse are skipped (as in + training). Deterministic under ``seed``. + """ + rng = random.Random(seed) + parsable = [r for r in records if parse_statements(r.openui or "")] + counts: Counter[int] = Counter() + attempted = 0 + for _ in range(samples): + if not parsable: + break + record = parsable[rng.randrange(len(parsable))] + statements = parse_statements(record.openui) + assert statements is not None + inventory = _inventory(record) + attempted += 1 + if rng.random() < stop_fraction: + counts[ACTION_STOP] += 1 + continue + k = rng.randint(1, max_chain) + current = statements + inverse: Edit | None = None + for _ in range(k): + step = space.sample_mutation( + current, + inventory, + rng, + inverse_action_weights=inverse_action_weights, + ) + if step is None: + break + current, inverse = step + if inverse is not None: + counts[inverse.action] += 1 + total = sum(counts.values()) + return { + "samples": attempted, + "n_targets": total, + "counts": { + ACTION_NAMES[a]: counts.get(a, 0) for a in range(len(ACTION_NAMES)) + }, + "shares": { + ACTION_NAMES[a]: (counts.get(a, 0) / total if total else 0.0) + for a in range(len(ACTION_NAMES)) + }, + "inverse_action_weights": ( + None + if inverse_action_weights is None + else { + ACTION_NAMES[a]: w for a, w in sorted(inverse_action_weights.items()) + } + ), + } + + +def applicable_add_exists( + statements: list, inventory: list[str], space: TreeEditSpace +) -> bool: + """True when at least one ADD edit is applicable to the state.""" + leaf_comps = [ + i for i, c in enumerate(space.components) if c in LEAF_COMPONENTS + ] + for stmt in range(len(statements)): + for comp in leaf_comps: + for slot in range(min(len(inventory), MAX_SLOTS)): + if space.apply( + statements, Edit(ACTION_ADD, stmt, comp, slot), inventory + ): + return True + return False + + +def _new_bucket() -> dict[str, Any]: + return { + "visited": Counter(), + "applicable": Counter(), + "selected": Counter(), + "dead": Counter(), + "rejection_reasons": Counter(), + "verifier_calls": 0, + "states_with_applicable_add": 0, + "states_recalled": 0, + } + + +def _add_proposals(bucket: dict[str, Any], proposals: list[dict[str, Any]]) -> None: + for p in proposals: + name = p["action_name"] + bucket["visited"][name] += 1 + if p["applicable"]: + bucket["applicable"][name] += 1 + if p["selected"]: + bucket["selected"][name] += 1 + if p["rejection_reason"] is not None: + bucket["dead"][name] += 1 + bucket["rejection_reasons"][p["rejection_reason"]] += 1 + + +def _add_recall( + bucket: dict[str, Any], + ev: dict[str, Any], + inventory: list[str], + space: TreeEditSpace, +) -> None: + """Exact applicable-ADD recall over the states this decode expanded.""" + proposals = ev.get("proposals", []) + visited_add_states = { + (p["step"], p["beam_row"]) + for p in proposals + if p["action"] == ACTION_ADD and p["applicable"] + } + for entry in ev.get("states", []): + statements = parse_statements(entry["source"]) + if statements is None: + continue + if not applicable_add_exists(statements, inventory, space): + continue + bucket["states_with_applicable_add"] += 1 + if (entry["step"], entry["beam_row"]) in visited_add_states: + bucket["states_recalled"] += 1 + + +def _finalize(bucket: dict[str, Any]) -> dict[str, Any]: + visited = sum(bucket["visited"].values()) + dead = sum(bucket["dead"].values()) + n_add_states = bucket["states_with_applicable_add"] + return { + "visited": dict(sorted(bucket["visited"].items())), + "applicable": dict(sorted(bucket["applicable"].items())), + "selected": dict(sorted(bucket["selected"].items())), + "rejection_reasons": dict(sorted(bucket["rejection_reasons"].items())), + "visited_total": visited, + "dead_total": dead, + "dead_candidate_rate": (dead / visited) if visited else None, + "visited_shares": { + name: (count / visited if visited else 0.0) + for name, count in sorted(bucket["visited"].items()) + }, + "verifier_calls": bucket["verifier_calls"], + "states_with_applicable_add": n_add_states, + "states_recalled": bucket["states_recalled"], + "applicable_add_recall": ( + bucket["states_recalled"] / n_add_states if n_add_states else None + ), + } + + +def decode_demand_distribution( + model: TreeEditDiffusionModel, + records: list[ExampleRecord], + *, + include_outputs: bool = False, +) -> dict[str, Any]: + """Decode every record and aggregate proposal telemetry by action — + overall, by source record, and by suite (split).""" + requests = [ + GenerationRequest( + prompt=r.prompt, + slot_contract=tuple(_inventory(r)), + design_md=r.design_md, + ) + for r in records + ] + outputs = model.generate_batch_requests(requests) + evidence = model.consume_generation_evidence() + overall = _new_bucket() + by_source: dict[str, Any] = {} + by_suite: dict[str, Any] = {} + for record, ev in zip(records, evidence): + inventory = _inventory(record) + proposals = ev.get("proposals", []) + overall["verifier_calls"] += ev.get("verifier_calls", 0) + _add_proposals(overall, proposals) + _add_recall(overall, ev, inventory, model.space) + src = by_source.setdefault(record.id, _new_bucket()) + _add_proposals(src, proposals) + _add_recall(src, ev, inventory, model.space) + suite = by_suite.setdefault(record.split or "unspecified", _new_bucket()) + _add_proposals(suite, proposals) + _add_recall(suite, ev, inventory, model.space) + result = { + "overall": _finalize(overall), + "by_source": {k: _finalize(v) for k, v in sorted(by_source.items())}, + "by_suite": {k: _finalize(v) for k, v in sorted(by_suite.items())}, + } + if include_outputs: + result["outputs"] = outputs + result["evidence"] = evidence + return result + + +def action_calibration( + training: dict[str, Any], demand_overall: dict[str, Any] +) -> dict[str, Any]: + """Per-action training-target share vs decode visited share + MAD.""" + target = training["shares"] + visited = demand_overall["visited_shares"] + rows = {} + mad_terms = [] + for name in ACTION_NAMES: + t = target.get(name, 0.0) + d = visited.get(name, 0.0) + rows[name] = {"target_share": t, "visited_share": d, "gap": d - t} + mad_terms.append(abs(d - t)) + return { + "per_action": rows, + "mean_abs_deviation": sum(mad_terms) / len(mad_terms), + } + + +def run_distribution_audit( + model: TreeEditDiffusionModel, + records: list[ExampleRecord], + *, + seed: int = 0, + samples: int = 400, + max_chain: int = 4, + inverse_action_weights: dict[int, float] | None = None, +) -> dict[str, Any]: + """Full audit: training-target vs decode-demand distributions, dead + candidates, applicable-ADD recall, calibration, and the preregistered + loss-reweighting prediction.""" + training = training_target_distribution( + records, + model.space, + seed=seed, + samples=samples, + max_chain=max_chain, + inverse_action_weights=inverse_action_weights, + ) + demand = decode_demand_distribution(model, records) + calib = action_calibration(training, demand["overall"]) + recall = demand["overall"]["applicable_add_recall"] + add_gap = calib["per_action"]["ADD"]["gap"] + predicted = bool( + (recall is not None and recall < ADD_RECALL_FLOOR) + or abs(add_gap) > ADD_SHARE_GAP + ) + return { + "training_targets": training, + "decode_demand": demand, + "calibration": calib, + "loss_reweighting_prediction": { + "predicted": predicted, + "rule": ( + f"predict reweighting iff applicable-ADD recall < " + f"{ADD_RECALL_FLOOR} or |ADD visited share - ADD target " + f"share| > {ADD_SHARE_GAP}" + ), + "add_share_gap": add_gap, + "applicable_add_recall": recall, + }, + } diff --git a/src/slm_training/models/tree_edit_diffusion.py b/src/slm_training/models/tree_edit_diffusion.py index 9d973f5b1..6b34b1e36 100644 --- a/src/slm_training/models/tree_edit_diffusion.py +++ b/src/slm_training/models/tree_edit_diffusion.py @@ -63,6 +63,64 @@ ACTION_BIND_PLACEHOLDER = 10 # (re)bind a leaf's slot to an inventory placeholder N_ACTIONS = 11 +ACTION_NAMES: tuple[str, ...] = ( + "STOP", + "REPLACE", + "ADD", + "REMOVE", + "ADD_CONTAINER", + "REMOVE_CONTAINER", + "INSERT_SUBTREE", + "REPLACE_SUBTREE", + "INSERT_STATEMENT", + "REPLACE_STATEMENT", + "BIND_PLACEHOLDER", +) +ACTION_IDS: dict[str, int] = {name: i for i, name in enumerate(ACTION_NAMES)} + +# SLM-310: mutation kinds that can realize a given *inverse* (supervised +# repair) action. sample_mutation reweights its forward-noise choice by +# sampling the inverse action from a declared distribution and mapping back +# to a mutation kind. STOP is never an inverse action (unreachable). +INVERSE_TO_MUTATION_KINDS: dict[int, tuple[int, ...]] = { + ACTION_REPLACE: (ACTION_REPLACE,), + ACTION_ADD: (ACTION_REMOVE,), + ACTION_REMOVE: (ACTION_ADD, ACTION_INSERT_STATEMENT), + ACTION_ADD_CONTAINER: (ACTION_REMOVE_CONTAINER,), + ACTION_REMOVE_CONTAINER: (ACTION_ADD_CONTAINER, ACTION_INSERT_SUBTREE), + ACTION_INSERT_SUBTREE: (ACTION_REMOVE_CONTAINER,), + ACTION_REPLACE_SUBTREE: (ACTION_REPLACE_SUBTREE,), + ACTION_REPLACE_STATEMENT: (ACTION_REPLACE_STATEMENT,), + ACTION_BIND_PLACEHOLDER: (ACTION_BIND_PLACEHOLDER,), +} + +# SLM-310: reason codes emitted by TreeEditSpace.apply (out-param ``reason``) +# so decode-time proposal telemetry can report WHY a candidate was dead. +REASON_INDEX_OUT_OF_RANGE = "index_out_of_range" +REASON_NO_OP = "no_op" +REASON_LEAF_CONTAINER_MISMATCH = "leaf_container_mismatch" +REASON_MAX_STMTS = "max_stmts" +REASON_PARENT_OR_COMP = "parent_or_comp_precondition" +REASON_SLOT_OUT_OF_RANGE = "slot_out_of_range" +REASON_NOT_REMOVABLE = "not_removable" +REASON_UNREFERENCED_LEAF = "unreferenced_leaf" +REASON_TARGET_OUT_OF_RANGE = "target_out_of_range" +REASON_NOT_CONTAINER = "not_container" +REASON_SUBTREE_NOT_LEAF_ONLY = "subtree_not_leaf_only" +REASON_UNREFERENCED_CONTAINER = "unreferenced_container" +REASON_PAYLOAD_OUT_OF_RANGE = "payload_out_of_range" +REASON_PAYLOAD_NOT_LEAF = "payload_not_leaf" +REASON_NOT_CANONICAL_SUBTREE = "not_canonical_subtree" +REASON_STATEMENT_INVALID = "statement_invalid" +REASON_TARGET_NOT_CANONICAL = "target_not_canonical" +REASON_NOT_BINDABLE = "not_bindable" +REASON_NOT_LEAF_COMPONENT = "not_leaf_component" +REASON_UNKNOWN_ACTION = "unknown_action" +REASON_PRE_VALIDATE_REJECTED = "pre_validate_rejected" +REASON_INVALID_RESULT = "invalid_result" +# Decode-side (not from apply): candidate produced an already-seen state. +REASON_DUPLICATE_STATE = "duplicate_state" + MAX_STMTS = 24 MAX_SLOTS = 16 @@ -283,6 +341,7 @@ def apply( edit: Edit, inventory: list[str], pre_validate: Callable[[list[Statement]], bool] | None = None, + reason: list[str] | None = None, ) -> list[Statement] | None: """Apply one edit; None when inapplicable or invalid (fail closed). @@ -291,7 +350,16 @@ def apply( reachability analyzer to skip already-visited states). It can only reject, never accept: every accepted state is still re-validated through the real parser. + + ``reason`` (SLM-310) is an optional out-list: when the edit is + rejected, exactly one machine-readable rejection code is appended + (``REASON_*`` constants). Acceptance appends nothing. """ + def _fail(code: str) -> None: + if reason is not None: + reason.append(code) + return None + if edit.action == ACTION_STOP: return [Statement(**vars(s)) for s in statements] working = [ @@ -300,26 +368,26 @@ def apply( ] if edit.action == ACTION_REPLACE: if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] new_comp = self.components[edit.comp] leaf_like = not target.has_list if leaf_like != (new_comp in LEAF_COMPONENTS): - return None + return _fail(REASON_LEAF_CONTAINER_MISMATCH) if target.comp == new_comp: - return None + return _fail(REASON_NO_OP) target.comp = new_comp elif edit.action == ACTION_ADD: if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) if len(working) >= MAX_STMTS: - return None + return _fail(REASON_MAX_STMTS) parent = working[edit.stmt] comp = self.components[edit.comp] if not parent.has_list or comp not in LEAF_COMPONENTS: - return None + return _fail(REASON_PARENT_OR_COMP) if not inventory or not (0 <= edit.slot < len(inventory)): - return None + return _fail(REASON_SLOT_OUT_OF_RANGE) placeholder = inventory[edit.slot] if not placeholder.startswith(":"): placeholder = f":{placeholder}" @@ -336,10 +404,10 @@ def apply( ) elif edit.action == ACTION_REMOVE: if not (0 <= edit.stmt < len(working)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] if target.has_list or target.name == "root": - return None + return _fail(REASON_NOT_REMOVABLE) referenced = False for other in working: if target.name in other.children: @@ -349,22 +417,22 @@ def apply( # Unreferenced UI leaves stay immutable (old behavior); V0.5 # pack statements are unreferenced by construction and are # removable (inverse of INSERT_STATEMENT). - return None + return _fail(REASON_UNREFERENCED_LEAF) working = [s for s in working if s.name != target.name] elif edit.action == ACTION_ADD_CONTAINER: # Preconditions: parent is a container, MAX_STMTS bound, comp is a # container. The minted container starts EMPTY (leaf-only subtree) # so REMOVE_CONTAINER is an exact safe inverse. if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) if len(working) >= MAX_STMTS: - return None + return _fail(REASON_MAX_STMTS) parent = working[edit.stmt] comp = self.components[edit.comp] if not parent.has_list or comp not in CONTAINER_COMPONENTS: - return None + return _fail(REASON_PARENT_OR_COMP) if not (0 <= edit.target < len(CONTAINER_RESTS)): - return None + return _fail(REASON_TARGET_OUT_OF_RANGE) name = self.fresh_name(working) parent.children.append(name) working.append( @@ -380,18 +448,18 @@ def apply( # shapes the container-creating actions mint, so removal restores # the prior state exactly. Leaf children are dropped with it. if not (0 <= edit.stmt < len(working)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] if not target.has_list or target.name == "root": - return None + return _fail(REASON_NOT_CONTAINER) by_name = {s.name: s for s in working} if any( by_name.get(child) is None or by_name[child].has_list for child in target.children ): - return None + return _fail(REASON_SUBTREE_NOT_LEAF_ONLY) if not any(target.name in other.children for other in working): - return None + return _fail(REASON_UNREFERENCED_CONTAINER) drop = {target.name, *target.children} working = [ Statement( @@ -412,22 +480,22 @@ def apply( # comp is a container, payload indexes a leaf component, slot is # in inventory, MAX_STMTS bound. Inverse: REMOVE_CONTAINER. if not (0 <= edit.stmt < len(working) and 0 <= edit.comp < len(self.components)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) if len(working) + 2 > MAX_STMTS: - return None + return _fail(REASON_MAX_STMTS) parent = working[edit.stmt] root_comp = self.components[edit.comp] if not parent.has_list or root_comp not in CONTAINER_COMPONENTS: - return None + return _fail(REASON_PARENT_OR_COMP) if not (0 <= edit.payload < len(self.components)): - return None + return _fail(REASON_PAYLOAD_OUT_OF_RANGE) leaf_comp = self.components[edit.payload] if leaf_comp not in LEAF_COMPONENTS: - return None + return _fail(REASON_PAYLOAD_NOT_LEAF) if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): - return None + return _fail(REASON_SLOT_OUT_OF_RANGE) if not (0 <= edit.target < len(CONTAINER_RESTS)): - return None + return _fail(REASON_TARGET_OUT_OF_RANGE) placeholder = self._placeholder(inventory, edit.slot) cname = self.fresh_name(working) lname = self.fresh_name( @@ -451,21 +519,21 @@ def apply( # The small-canonical-subtree precondition keeps the inverse # (restore old leaf comp + slot) expressible as the same action. if not (0 <= edit.stmt < len(working)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] if not target.has_list or len(target.children) != 1: - return None + return _fail(REASON_NOT_CANONICAL_SUBTREE) by_name = {s.name: s for s in working} leaf = by_name.get(target.children[0]) if leaf is None or leaf.has_list: - return None + return _fail(REASON_NOT_CANONICAL_SUBTREE) if not (0 <= edit.payload < len(self.components)): - return None + return _fail(REASON_PAYLOAD_OUT_OF_RANGE) leaf_comp = self.components[edit.payload] if leaf_comp not in LEAF_COMPONENTS: - return None + return _fail(REASON_PAYLOAD_NOT_LEAF) if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): - return None + return _fail(REASON_SLOT_OUT_OF_RANGE) placeholder = self._placeholder(inventory, edit.slot) leaf.comp = leaf_comp leaf.rest = json.dumps(placeholder, ensure_ascii=False) @@ -476,15 +544,15 @@ def apply( # Preconditions: MAX_STMTS bound, payload indexes V05_TEMPLATES. # Inverse: REMOVE (pack statements are unreferenced). if len(working) >= MAX_STMTS: - return None + return _fail(REASON_MAX_STMTS) if not (0 <= edit.payload < len(V05_TEMPLATES)): - return None + return _fail(REASON_PAYLOAD_OUT_OF_RANGE) comp, args = V05_TEMPLATES[edit.payload] candidate = Statement(self.fresh_v05_name(working, comp), comp, [], args, False) try: validate_output(candidate.render(), kind="statement") except Exception: # noqa: BLE001 - return None + return _fail(REASON_STATEMENT_INVALID) working.append(candidate) elif edit.action == ACTION_REPLACE_STATEMENT: # Swap one canonical V0.5 statement for another template. @@ -492,20 +560,20 @@ def apply( # (so the inverse — restore the old template — is expressible), # payload indexes V05_TEMPLATES, and the swap is a real change. if not (0 <= edit.stmt < len(working)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] if v05_template_index(target) is None: - return None + return _fail(REASON_TARGET_NOT_CANONICAL) if not (0 <= edit.payload < len(V05_TEMPLATES)): - return None + return _fail(REASON_PAYLOAD_OUT_OF_RANGE) comp, args = V05_TEMPLATES[edit.payload] if target.comp == comp and target.rest.strip() == args: - return None + return _fail(REASON_NO_OP) candidate = Statement(target.name, comp, [], args, False) try: validate_output(candidate.render(), kind="statement") except Exception: # noqa: BLE001 - return None + return _fail(REASON_STATEMENT_INVALID) working[edit.stmt] = candidate elif edit.action == ACTION_BIND_PLACEHOLDER: # Transactional declaration-plus-reference: (re)bind a leaf's slot @@ -513,23 +581,23 @@ def apply( # (non-root, non-container, leaf component), slot in inventory. # Inverse: BIND_PLACEHOLDER with the old slot index. if not (0 <= edit.stmt < len(working)): - return None + return _fail(REASON_INDEX_OUT_OF_RANGE) target = working[edit.stmt] if target.has_list or target.name == "root": - return None + return _fail(REASON_NOT_BINDABLE) if target.comp not in LEAF_COMPONENTS: - return None + return _fail(REASON_NOT_LEAF_COMPONENT) if not inventory or not (0 <= edit.slot < min(len(inventory), MAX_SLOTS)): - return None + return _fail(REASON_SLOT_OUT_OF_RANGE) placeholder = self._placeholder(inventory, edit.slot) target.rest = json.dumps(placeholder, ensure_ascii=False) else: - return None + return _fail(REASON_UNKNOWN_ACTION) if pre_validate is not None and not pre_validate(working): - return None + return _fail(REASON_PRE_VALIDATE_REJECTED) rendered = render_statements(working) if not _is_valid(rendered): - return None + return _fail(REASON_INVALID_RESULT) return working def sample_mutation( @@ -537,24 +605,45 @@ def sample_mutation( statements: list[Statement], inventory: list[str], rng: random.Random, + inverse_action_weights: dict[int, float] | None = None, ) -> tuple[list[Statement], Edit] | None: """One random validity-preserving mutation and the *inverse* edit - (the supervised repair step) — Kapur's forward process.""" + (the supervised repair step) — Kapur's forward process. + + ``inverse_action_weights`` (SLM-310) optionally declares a target + distribution over the *inverse* (repair) action: the inverse action + is sampled from the declared weights and mapped back to a mutation + kind that can realize it (``INVERSE_TO_MUTATION_KINDS``). Inverse + actions with no realizing mutation kind are skipped via the normal + retry loop. None = historical uniform-over-mutation-kinds behavior + (checkpoint / seed parity). + """ for _ in range(12): - kind = rng.choice( - ( - ACTION_REPLACE, - ACTION_ADD, - ACTION_REMOVE, - ACTION_ADD_CONTAINER, - ACTION_REMOVE_CONTAINER, - ACTION_INSERT_SUBTREE, - ACTION_REPLACE_SUBTREE, - ACTION_INSERT_STATEMENT, - ACTION_REPLACE_STATEMENT, - ACTION_BIND_PLACEHOLDER, + if inverse_action_weights: + population = sorted(inverse_action_weights) + inverse_action = rng.choices( + population, + weights=[inverse_action_weights[a] for a in population], + )[0] + kinds = INVERSE_TO_MUTATION_KINDS.get(inverse_action, ()) + if not kinds: + continue + kind = rng.choice(kinds) + else: + kind = rng.choice( + ( + ACTION_REPLACE, + ACTION_ADD, + ACTION_REMOVE, + ACTION_ADD_CONTAINER, + ACTION_REMOVE_CONTAINER, + ACTION_INSERT_SUBTREE, + ACTION_REPLACE_SUBTREE, + ACTION_INSERT_STATEMENT, + ACTION_REPLACE_STATEMENT, + ACTION_BIND_PLACEHOLDER, + ) ) - ) if kind == ACTION_REPLACE: idx = rng.randrange(len(statements)) stmt = statements[idx] @@ -884,6 +973,19 @@ class TreeEditDiffusionConfig: # (see ``from_checkpoint``) for behavior parity. value_label_mode: str = "bounded_distance" pairwise_progress_margin: float = 0.1 + # SLM-310 (LAR2-03): declared inverse-action distribution for the + # corruption sampler, mapping action NAME (``ACTION_NAMES``, e.g. "ADD") + # to a non-negative weight. Reweights sample_mutation's forward-noise + # choice toward mutation kinds whose inverse edit matches the declared + # distribution (e.g. ADD-balanced supervision); the gold corpus is never + # touched. None (default) = historical uniform behavior. + corruption_action_distribution: dict[str, float] | None = None + # SLM-310: STOP-slot accounting during decode. "legacy" (default, + # historical): every enumerated STOP proposal consumes an expand_per_state + # slot even when its frozen candidate is dropped as a duplicate. + # "corrected": STOP consumes a slot only when its frozen candidate is + # actually retained on the beam. + stop_slot_accounting: str = "legacy" # SLM-308: oracle depth/budget used for training-time value labels. The @@ -1085,6 +1187,28 @@ def _distance_label(self, statements, target, inventory, witness: int): upper_bound_witness=witness, ) + def _inverse_action_weights(self) -> dict[int, float] | None: + """SLM-310 corruption sampler: declared inverse-action weights keyed + by action id, or None for the historical uniform sampler.""" + declared = self.config.corruption_action_distribution + if not declared: + return None + weights: dict[int, float] = {} + for name, weight in declared.items(): + if name not in ACTION_IDS: + raise ValueError( + f"corruption_action_distribution names unknown action {name!r} " + f"(known: {sorted(ACTION_IDS)})" + ) + if weight < 0: + raise ValueError( + f"corruption_action_distribution weight for {name!r} is negative" + ) + weights[ACTION_IDS[name]] = float(weight) + if not any(weights.values()): + raise ValueError("corruption_action_distribution has no positive weight") + return weights + def forward(self, batch: list[ExampleRecord]) -> float: return float(self.training_loss(batch).detach().cpu()) @@ -1093,6 +1217,11 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: raise ValueError( f"unknown value_label_mode {self.config.value_label_mode!r}" ) + if self.config.stop_slot_accounting not in {"legacy", "corrected"}: + raise ValueError( + f"unknown stop_slot_accounting {self.config.stop_slot_accounting!r}" + ) + inverse_weights = self._inverse_action_weights() bounded_mode = self.config.value_label_mode == "bounded_distance" prompts: list[str] = [] states: list[str] = [] @@ -1133,7 +1262,10 @@ def training_loss(self, batch: list[ExampleRecord]) -> torch.Tensor: inverse: Edit | None = None applied = 0 for _ in range(k): - step = self.space.sample_mutation(current, inventory, self._rng) + step = self.space.sample_mutation( + current, inventory, self._rng, + inverse_action_weights=inverse_weights, + ) if step is None: break prev = current @@ -1391,8 +1523,24 @@ def _decode_one( seed = self._seed_state(inventory) if seed is None: return "", {"failure": "no_valid_seed"} + if self.config.stop_slot_accounting not in {"legacy", "corrected"}: + raise ValueError( + f"unknown stop_slot_accounting {self.config.stop_slot_accounting!r}" + ) beam: list[tuple[float, list[Statement], bool]] = [(0.0, seed, False)] evidence: dict[str, Any] = {"steps": 0, "expansions": 0, "kind": "tree_edit"} + # SLM-310: per-proposal reason-coded applicability telemetry. Every + # enumerated candidate the decode loop visits is recorded in + # deterministic (score-sorted) enumeration order with its head + # log-prob factor score, whether TreeEditSpace.apply accepts it, the + # rejection reason code when rejected, whether it consumed + # expand_per_state budget, and whether it was retained on the beam. + proposals: list[dict[str, Any]] = [] + # Expanded (live) states per step, so downstream audits can re-check + # full edit-space applicability on exactly the states decode visited. + states_log: list[dict[str, Any]] = [] + verifier_calls = 0 + corrected_stop = self.config.stop_slot_accounting == "corrected" for _ in range(self.config.max_search_steps): live = [entry for entry in beam if not entry[2]] if not live: @@ -1411,32 +1559,76 @@ def _decode_one( render_statements(s) for _, s, frozen in next_beam if frozen } for row, (_, statements, _) in enumerate(live): + states_log.append( + { + "step": evidence["steps"], + "beam_row": row, + "source": render_statements(statements), + } + ) candidates = self._enumerate_edits( out, row, len(statements), len(inventory) ) expanded = 0 - for _, edit in candidates: + for rank, (score, edit) in enumerate(candidates): if expanded >= self.config.expand_per_state: break + record: dict[str, Any] = { + "step": evidence["steps"], + "beam_row": row, + "rank": rank, + "action": edit.action, + "action_name": ACTION_NAMES[edit.action], + "score": float(score), + } + proposals.append(record) if edit.action == ACTION_STOP: text = render_statements(statements) - if text not in seen: + retained = text not in seen + record["applicable"] = True + record["rejection_reason"] = ( + None if retained else REASON_DUPLICATE_STATE + ) + record["selected"] = retained + if retained: seen.add(text) next_beam.append( (float(out["value"][row]), statements, True) ) - expanded += 1 + # STOP-slot accounting arms (SLM-310): legacy consumes + # a slot for every STOP proposal; corrected consumes + # one only when the frozen candidate is retained. + consumed = retained if corrected_stop else True + record["consumed_budget"] = consumed + if consumed: + expanded += 1 continue - child = self.space.apply(statements, edit, inventory) + reason: list[str] = [] + child = self.space.apply( + statements, edit, inventory, reason=reason + ) + verifier_calls += 1 if child is None: + record["applicable"] = False + record["rejection_reason"] = reason[0] + record["selected"] = False + record["consumed_budget"] = False continue text = render_statements(child) if text in seen: + record["applicable"] = True + record["rejection_reason"] = REASON_DUPLICATE_STATE + record["selected"] = False + record["consumed_budget"] = False continue seen.add(text) next_beam.append((float(out["value"][row]), child, False)) expanded += 1 evidence["expansions"] += 1 + record["applicable"] = True + record["rejection_reason"] = None + record["selected"] = True + record["consumed_budget"] = True if not next_beam: break # Re-score unfrozen children by the value head (Kapur's search @@ -1466,6 +1658,21 @@ def _decode_one( best = max(beam, key=lambda entry: entry[0]) evidence["value"] = float(best[0]) evidence["frozen"] = bool(best[2]) + evidence["proposals"] = proposals + evidence["states"] = states_log + evidence["verifier_calls"] = verifier_calls + evidence["proposal_summary"] = { + "visited": len(proposals), + "applicable": sum(1 for p in proposals if p["applicable"]), + "selected": sum(1 for p in proposals if p["selected"]), + "consumed_budget": sum(1 for p in proposals if p["consumed_budget"]), + "dead": sum( + 1 + for p in proposals + if not p["applicable"] + or p["rejection_reason"] == REASON_DUPLICATE_STATE + ), + } return render_statements(best[1]), evidence def generate_batch_requests(self, requests: list[GenerationRequest]) -> list[str]: diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 7a3a7c3bd..015d3f848 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -4480,7 +4480,7 @@ ] }, "harness.experiments.slm299_edit_reachability": { - "version": "v3", + "version": "v4", "kind": "harness", "paths": [ "src/slm_training/harnesses/experiments/slm299_edit_reachability.py", @@ -4494,6 +4494,11 @@ "tests/test_models/test_tree_edit_diffusion.py" ], "history": [ + { + "version": "v4", + "date": "2026-07-24", + "note": "SLM-310: tree_edit_diffusion gains reason-coded apply rejections, per-proposal decode telemetry (applicability/rejection/budget), corruption_action_distribution + stop_slot_accounting config knobs (defaults = historical behavior, checkpoint parity); test_tree_edit_diffusion unchanged semantics" + }, { "version": "v3", "date": "2026-07-24", @@ -4530,25 +4535,6 @@ } ] }, - "harness.experiments.slm301_prompt_observability": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm301_prompt_observability.py", - "scripts/run_slm301_prompt_observability.py", - "tests/test_harnesses/experiments/test_slm301_prompt_observability.py", - "docs/design/iter-slm301-prompt-observability-20260724.json", - "docs/design/iter-slm301-prompt-observability-20260724.md", - "docs/design/slm301-slot-observability-audit-20260724.json" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-301 LAR1-04 prompt-observability arms (as-is vs +inventory) with phase-0 coverage classification, AR/X22 baselines, paired predeclared-delta report" - } - ] - }, "harness.experiments.slm303_decode_budget_audit": { "version": "v1", "kind": "harness", @@ -6312,7 +6298,7 @@ { "version": "v223", "date": "2026-07-22", - "note": "Derive component-array boundaries, nonempty typed children, available-content closure, typed binder compatibility, and acyclic references from schema so constrained decode cannot enter impossible continuations" + "note": "Derive component-array boundaries, nonempty typed children, available-content closure, typed binder compatibility, and acyclic references from schema so constrained decode cannot enter impossible continuations" }, { "version": "v222", @@ -7837,6 +7823,24 @@ "note": "initial registration; byte-preserving archived harness replay records" } ] + }, + "harness.experiments.slm310_action_alignment": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm310_action_alignment.py", + "scripts/run_slm310_action_alignment.py", + "tests/test_harnesses/experiments/test_slm310_action_alignment.py", + "docs/design/iter-slm310-action-alignment-20260724.json", + "docs/design/iter-slm310-action-alignment-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-310 LAR2-03 action alignment: reason-coded proposal telemetry, train-target vs decode-demand distribution audit, declared inverse-action corruption sampler, legacy/corrected STOP-slot accounting arms, matched 2x2 fixture experiment with preregistered thresholds" + } + ] } } } diff --git a/tests/test_harnesses/experiments/test_slm310_action_alignment.py b/tests/test_harnesses/experiments/test_slm310_action_alignment.py new file mode 100644 index 000000000..7dc01e9b8 --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm310_action_alignment.py @@ -0,0 +1,405 @@ +"""SLM-310 (LAR2-03): action-alignment invariants for the X22 tree-edit model.""" + +from __future__ import annotations + +import json +import random + +import pytest + +torch = pytest.importorskip("torch") + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm310_action_alignment import ( + action_calibration, + decode_demand_distribution, + run_distribution_audit, + training_target_distribution, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_ADD, + ACTION_BIND_PLACEHOLDER, + ACTION_IDS, + ACTION_REMOVE, + ACTION_STOP, + Edit, + REASON_DUPLICATE_STATE, + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + TreeEditSpace, + parse_statements, +) +from scripts.run_slm310_action_alignment import CELLS + +PROGRAM = ( + 'root = Stack([inline_card, cta], "column")\n' + "inline_card = Card([title])\n" + 'title = TextContent(":hero.title")\n' + 'cta = Button(":cta.label")' +) +INVENTORY = [":hero.title", ":cta.label"] + +PROGRAMS = [ + PROGRAM, + 'root = Stack([t], "column")\nt = TextContent(":body")', + 'root = Form([i, s], "column")\ni = TextInput(":form.email")\ns = Button(":form.submit")', +] +INVENTORIES = [INVENTORY, [":body"], [":form.email", ":form.submit"]] + + +def _records() -> list[ExampleRecord]: + return [ + ExampleRecord( + id=f"fx_{i}", + prompt=f"fixture prompt {i}", + openui=openui, + placeholders=list(inv), + split="train", + ) + for i, (openui, inv) in enumerate(zip(PROGRAMS, INVENTORIES)) + ] + + +def _model(**overrides) -> TreeEditDiffusionModel: + base = dict( + d_model=32, + n_heads=4, + context_layers=1, + denoiser_layers=1, + beam_width=2, + expand_per_state=2, + max_search_steps=3, + seed=3, + value_label_mode="mutation_count", + ) + base.update(overrides) + cfg = TreeEditDiffusionConfig(**base) + torch.manual_seed(cfg.seed) # deterministic init across test order + return TreeEditDiffusionModel.from_records(_records(), config=cfg, device="cpu") + + +def _evidence(model: TreeEditDiffusionModel) -> list[dict]: + from slm_training.harnesses.model_build.plugin import GenerationRequest + + model.generate_batch_requests( + [ + GenerationRequest(prompt=r.prompt, slot_contract=tuple(r.placeholders)) + for r in _records() + ] + ) + return model.consume_generation_evidence() + + +# --- reason-coded applicability --------------------------------------------- + + +def test_apply_reports_reason_codes() -> None: + space = TreeEditSpace() + base = parse_statements(PROGRAM) + assert base is not None + # Out-of-range statement index. + reason: list[str] = [] + assert space.apply(base, Edit(ACTION_REMOVE, 99), INVENTORY, reason=reason) is None + assert reason == ["index_out_of_range"] + # Leaf/container mismatch on REPLACE (root is a container, leaf comp given). + reason = [] + leaf_comp = space.comp_index["TextContent"] + assert space.apply(base, Edit(1, 0, leaf_comp), INVENTORY, reason=reason) is None + assert reason == ["leaf_container_mismatch"] + # BIND_PLACEHOLDER on the root container. + reason = [] + assert ( + space.apply(base, Edit(ACTION_BIND_PLACEHOLDER, 0, slot=0), INVENTORY, reason=reason) + is None + ) + assert reason == ["not_bindable"] + # Acceptance appends nothing. + reason = [] + assert space.apply(base, Edit(ACTION_REMOVE, 3), INVENTORY, reason=reason) is not None + assert reason == [] + + +def test_proposal_telemetry_reason_coded_and_deterministic() -> None: + # Wide expansion budget so the decode loop visits inapplicable + # (dead) candidates, not only top-scoring applicable ones. + model = _model(expand_per_state=8, beam_width=3, max_search_steps=4) + first = _evidence(model) + second = _evidence(model) + # Deterministic: identical proposals across identical decodes. + assert [e.get("proposals") for e in first] == [e.get("proposals") for e in second] + saw_rejection = False + saw_reason_codes: set[str] = set() + for ev in first: + proposals = ev.get("proposals") + assert proposals, "every decode records proposal telemetry" + assert ev["verifier_calls"] == sum( + 1 for p in proposals if p["action"] != ACTION_STOP + ) + for p in proposals: + assert set(p) >= { + "step", + "beam_row", + "rank", + "action", + "action_name", + "score", + "applicable", + "rejection_reason", + "selected", + "consumed_budget", + } + # Deterministic order: ranks are strictly increasing per state. + if p["applicable"]: + assert p["rejection_reason"] in (None, REASON_DUPLICATE_STATE) + else: + assert isinstance(p["rejection_reason"], str) + saw_rejection = True + saw_reason_codes.add(p["rejection_reason"]) + # Budget accounting invariants: consumed implies applicable, and + # every selected proposal consumed budget. + if p["consumed_budget"]: + assert p["applicable"] + if p["selected"]: + assert p["consumed_budget"] + # Summary is internally consistent. + summary = ev["proposal_summary"] + assert summary["visited"] == len(proposals) + assert summary["applicable"] == sum(1 for p in proposals if p["applicable"]) + assert saw_rejection, "fixture decode must exercise dead candidates" + assert saw_reason_codes <= { + "index_out_of_range", + "no_op", + "leaf_container_mismatch", + "max_stmts", + "parent_or_comp_precondition", + "slot_out_of_range", + "not_removable", + "unreferenced_leaf", + "target_out_of_range", + "not_container", + "subtree_not_leaf_only", + "unreferenced_container", + "payload_out_of_range", + "payload_not_leaf", + "not_canonical_subtree", + "statement_invalid", + "target_not_canonical", + "not_bindable", + "not_leaf_component", + "unknown_action", + "pre_validate_rejected", + "invalid_result", + } + + +# --- distribution audit math --------------------------------------------------- + + +def test_training_target_distribution_math() -> None: + space = TreeEditSpace() + dist = training_target_distribution( + _records(), space, seed=5, samples=120, max_chain=3 + ) + assert dist["n_targets"] == sum(dist["counts"].values()) + assert abs(sum(dist["shares"].values()) - 1.0) < 1e-9 + assert dist["counts"]["STOP"] > 0 # stop_fraction branch exercised + # Deterministic. + again = training_target_distribution( + _records(), space, seed=5, samples=120, max_chain=3 + ) + assert again["counts"] == dist["counts"] + + +def test_decode_demand_audit_math() -> None: + model = _model() + demand = decode_demand_distribution(model, _records()) + overall = demand["overall"] + assert overall["visited_total"] > 0 + assert overall["verifier_calls"] > 0 + assert 0.0 <= overall["dead_candidate_rate"] <= 1.0 + assert abs(sum(overall["visited_shares"].values()) - 1.0) < 1e-6 + assert set(demand["by_source"]) == {r.id for r in _records()} + assert set(demand["by_suite"]) == {"train"} + # Dead total = visited - (no-reason proposals). + assert overall["dead_total"] <= overall["visited_total"] + # Calibration rows cover every action and MAD is the mean |gap|. + calib = action_calibration( + training_target_distribution(_records(), model.space, seed=5, samples=60), + overall, + ) + assert len(calib["per_action"]) == 11 + gaps = [abs(r["gap"]) for r in calib["per_action"].values()] + assert abs(calib["mean_abs_deviation"] - sum(gaps) / len(gaps)) < 1e-12 + # Full audit predicts via the preregistered rule and stays honest. + audit = run_distribution_audit(model, _records(), seed=5, samples=60) + pred = audit["loss_reweighting_prediction"] + assert isinstance(pred["predicted"], bool) + assert pred["applicable_add_recall"] == overall["applicable_add_recall"] + + +# --- corruption sampler --------------------------------------------------------- + + +def test_corruption_sampler_hits_declared_distribution() -> None: + space = TreeEditSpace() + statements = parse_statements(PROGRAM) + assert statements is not None + # Degenerate declared distribution: all weight on ADD inverses. + rng = random.Random(13) + inverses = [] + for _ in range(40): + step = space.sample_mutation( + statements, INVENTORY, rng, + inverse_action_weights={ACTION_ADD: 1.0}, + ) + if step is None: + continue + inverses.append(step[1].action) + assert inverses, "sampler must still produce mutations" + assert set(inverses) == {ACTION_ADD} + # Declared ADD-balanced distribution lands near the declared share. + declared = {ACTION_IDS[n]: w for n, w in { + "REPLACE": 1.0, "ADD": 4.0, "REMOVE": 1.0, "ADD_CONTAINER": 1.0, + "REMOVE_CONTAINER": 1.0, "INSERT_SUBTREE": 1.0, "REPLACE_SUBTREE": 1.0, + "REPLACE_STATEMENT": 1.0, "BIND_PLACEHOLDER": 1.0, + }.items()} + dist = training_target_distribution( + _records(), space, seed=7, samples=600, max_chain=3, + inverse_action_weights=declared, + ) + default = training_target_distribution( + _records(), space, seed=7, samples=600, max_chain=3 + ) + add_share = dist["shares"]["ADD"] + assert add_share > default["shares"]["ADD"] + 0.10 + # Declared ADD weight is 4/12 of inverse mass; with 20% STOP targets the + # realized share should sit near 0.8 * 4/12 ≈ 0.27 (loose tolerance for + # retry-loop fallback and REMOVE-mutation applicability). + assert abs(add_share - 0.8 * 4 / 12) < 0.12 + + +def test_corruption_sampler_leaves_gold_corpus_untouched() -> None: + records = _records() + before = [r.openui for r in records] + model = _model( + corruption_action_distribution={"ADD": 4.0, "REPLACE": 1.0, "REMOVE": 1.0} + ) + loss = model.training_loss(records) + assert torch.isfinite(loss) + assert [r.openui for r in records] == before + # Default off: config without the knob keeps the historical sampler. + assert _model()._inverse_action_weights() is None + # Unknown action names fail closed. + bad = _model(corruption_action_distribution={"NOT_AN_ACTION": 1.0}) + with pytest.raises(ValueError, match="unknown action"): + bad.training_loss(records) + + +# --- STOP-slot accounting arms --------------------------------------------------- + + +def test_stop_slot_accounting_arms_differ_exactly_as_specified() -> None: + legacy = _model(stop_slot_accounting="legacy") + corrected = _model(stop_slot_accounting="corrected") + leg_ev = _evidence(legacy) + cor_ev = _evidence(corrected) + # Deterministic per arm. + assert [e.get("proposals") for e in leg_ev] == [ + e.get("proposals") for e in _evidence(legacy) + ] + assert [e.get("proposals") for e in cor_ev] == [ + e.get("proposals") for e in _evidence(corrected) + ] + + def stop_stats(evs): + visited = consumed = retained = 0 + for ev in evs: + for p in ev.get("proposals", []): + if p["action"] != ACTION_STOP: + continue + visited += 1 + consumed += 1 if p["consumed_budget"] else 0 + retained += 1 if p["selected"] else 0 + return visited, consumed, retained + + lv, lc, lr = stop_stats(leg_ev) + cv, cc, cr = stop_stats(cor_ev) + assert lv > 0, "fixture decode must visit STOP proposals" + # Legacy: every STOP consumes a slot, retained or not. + assert lc == lv + # Corrected: STOP consumes a slot exactly when its frozen candidate is + # retained. + assert cc == cr + assert cc <= lc + # Unknown arm values fail closed. + bad = _model(stop_slot_accounting="bogus") + with pytest.raises(ValueError, match="stop_slot_accounting"): + _evidence(bad) + + +def test_lever_isolation_single_factor_per_arm() -> None: + def diff(x, y): + return [k for k in ("corruption", "stop") if CELLS[x][k] != CELLS[y][k]] + + assert diff("cell_b", "cell_a") == ["corruption"] + assert diff("cell_c", "cell_a") == ["stop"] + assert diff("cell_d", "cell_b") == ["stop"] + assert diff("cell_d", "cell_c") == ["corruption"] + + +# --- checkpoint parity ----------------------------------------------------------- + + +def test_checkpoint_parity_with_new_additive_fields(tmp_path) -> None: + records = _records() + model = _model(corruption_action_distribution={"ADD": 2.0, "REMOVE": 1.0}) + path = tmp_path / "ckpt.pt" + model.save(path) + loaded = TreeEditDiffusionModel.from_checkpoint(path, device="cpu") + assert loaded.config.corruption_action_distribution == { + "ADD": 2.0, + "REMOVE": 1.0, + } + assert loaded.config.stop_slot_accounting == "legacy" + # Simulate a pre-SLM-310 checkpoint: drop the new config keys entirely. + payload = torch.load(path, map_location="cpu", weights_only=False) + payload["config"].pop("corruption_action_distribution") + payload["config"].pop("stop_slot_accounting") + old_path = tmp_path / "ckpt_old.pt" + torch.save(payload, old_path) + (tmp_path / "ckpt_old.tokenizer.json").write_text( + (tmp_path / "ckpt.tokenizer.json").read_text(encoding="utf-8"), + encoding="utf-8", + ) + old = TreeEditDiffusionModel.from_checkpoint(old_path, device="cpu") + # Pre-field checkpoints get the historical defaults: uniform sampler, + # legacy STOP accounting. + assert old.config.corruption_action_distribution is None + assert old.config.stop_slot_accounting == "legacy" + # Decode parity: the old-loaded model reproduces the default model. + from slm_training.harnesses.model_build.plugin import GenerationRequest + + requests = [ + GenerationRequest(prompt=r.prompt, slot_contract=tuple(r.placeholders)) + for r in records + ] + default_model = _model() + assert old.generate_batch_requests(requests) == default_model.generate_batch_requests( + requests + ) + + +def test_metadata_not_polluted(tmp_path) -> None: + # Proposal telemetry stays on evidence; meta.json schema unchanged. + model = _model() + path = tmp_path / "ckpt.pt" + model.save(path) + meta = json.loads(path.with_suffix(".meta.json").read_text(encoding="utf-8")) + assert set(meta) == { + "kind", + "format_version", + "tokenizer", + "vocab_size", + "parameter_count", + "serialized_weight_bytes", + } From 0dd815193a4fccb41f229751393ed18a989718d7 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:15:09 -0500 Subject: [PATCH 07/18] SLM-312: seedward and on-policy state sources with matched mixtures - State-source harness: gold_only (existing corruption chain), seedward (offline oracle-guided walk from the seed toward gold, strictly distance-decreasing valid intermediates), on_policy (immutable content-addressed beam-trajectory snapshots: wrong states, verifier failures, abstentions); explicit provenance + gold-visibility policy per row; sha256 rows + tamper-evident manifest; fail-closed leakage guards (train vs held-out AST fingerprints). - Matched {gold_only, seedward, on_policy, mixed} arms with predeclared weights/caps, identical model/steps/optimizer/seeds; evaluated on held-out seed trajectories. Verdict: rejected per preregistered primary gate (beam regret improvement 0.0 < 0.05); secondary signal (value rank corr -0.258 -> +0.258 for non-gold arms) recorded as wiring evidence only. - 9 new tests; slm312 component v1. --- .../iter-slm312-state-sources-20260724.json | 468 +++++++ .../iter-slm312-state-sources-20260724.md | 40 + scripts/run_slm312_state_sources.py | 526 ++++++++ .../experiments/slm312_state_sources.py | 1150 +++++++++++++++++ src/slm_training/resources/versions.json | 18 + .../experiments/test_slm312_state_sources.py | 310 +++++ 6 files changed, 2512 insertions(+) create mode 100644 docs/design/iter-slm312-state-sources-20260724.json create mode 100644 docs/design/iter-slm312-state-sources-20260724.md create mode 100644 scripts/run_slm312_state_sources.py create mode 100644 src/slm_training/harnesses/experiments/slm312_state_sources.py create mode 100644 tests/test_harnesses/experiments/test_slm312_state_sources.py diff --git a/docs/design/iter-slm312-state-sources-20260724.json b/docs/design/iter-slm312-state-sources-20260724.json new file mode 100644 index 000000000..9f68bebfe --- /dev/null +++ b/docs/design/iter-slm312-state-sources-20260724.json @@ -0,0 +1,468 @@ +{ + "experiment": "slm312-state-sources", + "issue": "SLM-312", + "question": "Does closing the gold-corruption vs seed-rollout distribution gap (seedward + on-policy state sources) improve repair more than extra offline near-gold training at equal budget?", + "preregistered_thresholds": { + "beam_regret_improvement_min": 0.05, + "rank_correlation_degradation_max": 0.05, + "comparison": "best non-gold arm (seedward/on_policy/mixed) vs gold_only", + "verdict_rule": "distribution_gap_closed iff best non-gold arm improves beam regret >= 0.05 AND degrades rank correlation by <= 0.05, else rejected" + }, + "preregistered_arms": { + "gold_only": { + "gold_only": 1.0 + }, + "seedward": { + "seedward": 1.0 + }, + "on_policy": { + "on_policy": 1.0 + }, + "mixed": { + "gold_only": 0.34, + "seedward": 0.33, + "on_policy": 0.33 + } + }, + "config": { + "steps": 4, + "batch_size": 4, + "per_arm_budget": 24, + "seed": 0, + "n_train_records": 6, + "n_held_out_records": 2, + "suite_id": "slm312-fixture-v1", + "fixture_config_overrides": { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_search_steps": 8, + "beam_width": 3, + "expand_per_state": 3 + }, + "on_policy_max_states_per_record": 3, + "torch_num_threads": 1 + }, + "snapshot": { + "path": "outputs/slm312/state_snapshot.jsonl", + "manifest": { + "snapshot_schema": "slm312-state-snapshot/v1", + "snapshot_id": "slm312-state-sources-fixture", + "config": { + "suite_id": "slm312-fixture-v1", + "arms": { + "gold_only": { + "gold_only": 1.0 + }, + "seedward": { + "seedward": 1.0 + }, + "on_policy": { + "on_policy": 1.0 + }, + "mixed": { + "gold_only": 0.34, + "seedward": 0.33, + "on_policy": 0.33 + } + }, + "per_arm_budget": 24, + "steps": 4, + "batch_size": 4, + "seed": 0, + "held_out_fingerprints": [ + "8008d5caebe08c70e2ccc1a558d30294c8d3c09e334f402b56ef960caab7c034", + "b5d4c7824bfcd66ea60732a53d5de59068366a1cd92cc92b58a13c56c23b3dfe", + "d12577927c9c7c1e1afd0b86f726611be9a9e901a9269be41d0c19bf3eb22826", + "e18b0d82658d1117f34b7704c2537a4112e4c5f3ecd0cd7dc1c6ac7873e4cf9b", + "fb34d4793d23dcc327c1677c10bac6d3ca338308a9b96307371ea0cff6325e44" + ], + "dropped_leak_collisions": [ + { + "record_id": "fixture_01", + "source": "gold_only", + "kind": "near_gold", + "fingerprint": "d12577927c9c7c1e1afd0b86f726611be9a9e901a9269be41d0c19bf3eb22826", + "reason": "held_out_fingerprint_collision" + } + ] + }, + "n_rows": 43, + "row_sha256": [ + "e4c806299e446dd8c66e14488e9ea4f3e7f4e3df53edf8495226594fb646802a", + "45ca113c57daed51731f2316082d801e38557e7ef45ed75933296ef86aaf1a76", + "3a15f53adb710b15b13fca9073022baa2fc15b4c6901ea9e62be013f878c3100", + "6dd53388dde698e988e7d1f5a626e135dbc8e47b1aaf10de40bc2897ae80c43d", + "468851f57f3ac185df3ecb4061ebc56f790d321fd1b6fb561ec9d681e449184e", + "bfacacc8e17be88bf6025280ba01e1382165ed9fb1fe4770dfcf178a50c6fe77", + "bc5495c95a9cc193772dd0bfd66a6f46df6afeb34101b5443d22df1e2d13e58b", + "3cf4cd1f147b61fd819a956b459cc7993b7d2e1db9b45ca9d99c074d19bd6f69", + "541329e8a4f741bd5e2c165d00de59dd73f47992bcb42dd3639651b30f8abbf0", + "3253c262988b27d635204c7f825c684c52e2942d85da53093f2c8406da76d212", + "d19ce94773aa0fe4c672a214abe71aad6194052a2061315efc6ced5d666a92d0", + "e3824aa7db17af0dfa4e4c0069c464e5dce26cbf78c0c45364c101bb3c111777", + "d63ce3d52c05f2bc41081c2f4c7b280359f4666014f21df40e66b3445e4a9764", + "b7d17e0e6d789c47be762254e6359e2d30a13953d33c55d70d1e99b797a196a0", + "9d19b5dff62d5c37f7c514b6b9c222978114ccba63a5b40a7e9248f7f2ab7970", + "3e77f595f255fb0dfb3a77fa06406d75ce43c2bceb2a0f6d7c09bb1daac63ff3", + "7b7ec2845c0f96d71d06e1055c46dbf52473a7373e419589b2eb725470b216be", + "73082b08c6aadb30c269e27d473ef310d613bda05f75df462cc3b8ab24532c17", + "9b1c394db2db6f1f53f4aa5e7a65b84d6632603dced37ae9b00aa27fc71f56ff", + "b81f2c4618f162aa60d6000d9165eed6964e339372e1859ac18855ecfc744d1c", + "e16506ef496224edf4fcbda3f02e903f9d2417893821e5ab160125b9320ed02d", + "ce9e9b3ca03634fbae5909ab778176a36d57efe65b63d65eddff7b7a0c479e62", + "cab1651e429d37beda52a0945f2b69ff702e611b1e047bf7e6cf0a7137dc8ab1", + "2d1058b70c7630ff6be864d4c2ea9be99d720c2d110b33fd50d97292e274b056", + "e0577d5ee84d94ac7d166efa259f77f552f26cbb16a3d4b06e33aa172782426c", + "3444ac129fdbd1cc80c5921dc41d0e0952358baad8d1d9d34cc5ad2bcc033df9", + "46e4a474ca7130324949b26bcb889f3d598fe7f44b6f232e23f0bfaac3577dab", + "c4db64855ca8c7335df1b4097a2675b78740b141c97c656cab153f308c412072", + "c7e474e54c16bfc6122d6e975a920d1d3a57ac722b367266b9a43c5788c38739", + "4608be6d174bfba6e0423f5924e94a57907a94d009d7ecb23463ac67a158c64b", + "5bccd49029d79b60b40fa6097ab4feb734cc97d4dc1dca8f93286f332d7dc67c", + "c71f33b629b18e4f28b512f0102de4103a4a5240dc0f9bac39fc559346742ee5", + "d2c75520ce823ad8b70e23df7a6b679a398fe8aad1bbf1eb44412d791ab638fe", + "fb47281652480bf7210aeb4ed2c4d7dae4892d091ee9fd13eb76b142a43851e6", + "515d2607be5696e82e109188d4e7c30bd88d51384febf38a465ce223b74c4705", + "9a42b62454c5a568b47d287a25697c82e569d852a6bf2baee472f46bac2d0cd3", + "51c5d01b01be5da4be2a5ac3b42915dfcea0426bc64dade805dd1b391e8fc2c7", + "7f328c9728482b7b2df76810f15fed1f2569142e37f541962b624242120be1ed", + "3798721b3c9c21b3ecba164943a9db6058db8edfffc8f8775cf1b05fdc636fee", + "b7f1430f244943bda8efa33753e0630c44cf1e4eae14b0cf5a3ffe01a928ef10", + "084b655bca1975150f50af36ec63e372cad6b6b82baeb5b29f4412e381a8c1e2", + "54a2405447bc92161495ac9937b925a6b97e82e74d380a3efae04566e8f49985", + "e71f2adb4ac78b3e9efdfbbc628c14482dd6246c494ed495538c1f33b8809654" + ], + "manifest_sha256": "1c5fc80b721b12f4c6541749cade9dd05d926051b682217073d86fa26973fe01" + }, + "coverage": { + "rows": 43, + "unique_fingerprints": 37, + "duplicate_rate": 0.13953488372093026, + "per_source": { + "gold_only": { + "rows": 17, + "unique_fingerprints": 17 + }, + "on_policy": { + "rows": 18, + "unique_fingerprints": 18 + }, + "seedward": { + "rows": 8, + "unique_fingerprints": 8 + } + } + } + }, + "oracle_cost": { + "n_states": 6, + "n_unknown": 2, + "unknown_coverage": 0.3333333333333333, + "n_measurable": 4, + "wall_ms_total": 8382.73887700052, + "ms_per_state": 1397.1231461667533, + "cache": { + "label_hits": 0, + "label_misses": 6, + "map_builds": 2, + "target_maps": 2, + "labels": 6, + "map_nodes": 16 + } + }, + "arms": { + "gold_only": { + "n_rows": 17, + "coverage": { + "rows": 17, + "unique_fingerprints": 17, + "duplicate_rate": 0.0, + "per_source": { + "gold_only": { + "rows": 17, + "unique_fingerprints": 17 + } + } + }, + "metrics": { + "n_measurable": 4, + "rank_correlation": -0.2581988897471611, + "concordance": 0.3333333333333333, + "brier": 0.03396308228040912, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 1, + "mean_predicted_value": 0.6047580242156982, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 3, + "mean_predicted_value": 0.6053926746050516, + "mean_target": 0.75 + } + ], + "beam_regret_mean": 0.0, + "beam_regret_n": 1, + "beam_regret_excluded": 3, + "by_origin": {} + }, + "valid_final": { + "n": 2, + "valid_final_rate": 1.0, + "finals": [ + { + "record_id": "fixture_06", + "valid": true, + "steps": 4, + "verifier_calls": 698 + }, + { + "record_id": "fixture_07", + "valid": true, + "steps": 5, + "verifier_calls": 710 + } + ] + }, + "training_metrics": { + "action": 2.2585463523864746, + "stmt": 2.9041709899902344, + "comp": 1.8313002586364746, + "slot": 3.4643394947052, + "value": 0.07977510988712311, + "pairwise_progress": 0.0953509584069252, + "value_unknown_excluded": 2.0, + "skipped": 0.0, + "rows": 4.0 + } + }, + "mixed": { + "n_rows": 24, + "coverage": { + "rows": 24, + "unique_fingerprints": 24, + "duplicate_rate": 0.0, + "per_source": { + "gold_only": { + "rows": 8, + "unique_fingerprints": 8 + }, + "on_policy": { + "rows": 8, + "unique_fingerprints": 8 + }, + "seedward": { + "rows": 8, + "unique_fingerprints": 8 + } + } + }, + "metrics": { + "n_measurable": 4, + "rank_correlation": 0.2581988897471611, + "concordance": 0.6666666666666666, + "brier": 0.006841517524175167, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 1, + "mean_predicted_value": 0.7193946242332458, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 3, + "mean_predicted_value": 0.7179443836212158, + "mean_target": 0.75 + } + ], + "beam_regret_mean": 0.0, + "beam_regret_n": 1, + "beam_regret_excluded": 3, + "by_origin": {} + }, + "valid_final": { + "n": 2, + "valid_final_rate": 1.0, + "finals": [ + { + "record_id": "fixture_06", + "valid": true, + "steps": 8, + "verifier_calls": 887 + }, + { + "record_id": "fixture_07", + "valid": true, + "steps": 8, + "verifier_calls": 1816 + } + ] + }, + "training_metrics": { + "action": 2.153684139251709, + "stmt": 2.635714054107666, + "comp": 1.7767260074615479, + "slot": 1.1594464778900146, + "value": 0.0398101769387722, + "pairwise_progress": 0.1034640446305275, + "value_unknown_excluded": 3.0, + "skipped": 0.0, + "rows": 4.0 + } + }, + "on_policy": { + "n_rows": 18, + "coverage": { + "rows": 18, + "unique_fingerprints": 18, + "duplicate_rate": 0.0, + "per_source": { + "on_policy": { + "rows": 18, + "unique_fingerprints": 18 + } + } + }, + "metrics": { + "n_measurable": 4, + "rank_correlation": 0.2581988897471611, + "concordance": 0.6666666666666666, + "brier": 0.014381557280683133, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 1, + "mean_predicted_value": 0.6760445833206177, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 3, + "mean_predicted_value": 0.6730026205380758, + "mean_target": 0.75 + } + ], + "beam_regret_mean": 0.0, + "beam_regret_n": 1, + "beam_regret_excluded": 3, + "by_origin": {} + }, + "valid_final": { + "n": 2, + "valid_final_rate": 1.0, + "finals": [ + { + "record_id": "fixture_06", + "valid": true, + "steps": 8, + "verifier_calls": 2598 + }, + { + "record_id": "fixture_07", + "valid": true, + "steps": 8, + "verifier_calls": 2598 + } + ] + }, + "training_metrics": { + "action": 2.603577136993408, + "stmt": 3.4750404357910156, + "value": 0.08445368707180023, + "pairwise_progress": 0.1012253537774086, + "value_unknown_excluded": 0.0, + "skipped": 0.0, + "rows": 4.0 + } + }, + "seedward": { + "n_rows": 8, + "coverage": { + "rows": 8, + "unique_fingerprints": 8, + "duplicate_rate": 0.0, + "per_source": { + "seedward": { + "rows": 8, + "unique_fingerprints": 8 + } + } + }, + "metrics": { + "n_measurable": 4, + "rank_correlation": 0.2581988897471611, + "concordance": 0.6666666666666666, + "brier": 0.03013388784179849, + "calibration_bins": [ + { + "distance_bin": 1, + "n": 1, + "mean_predicted_value": 0.6180281043052673, + "mean_target": 0.875 + }, + { + "distance_bin": 2, + "n": 3, + "mean_predicted_value": 0.6153192520141602, + "mean_target": 0.75 + } + ], + "beam_regret_mean": null, + "beam_regret_n": 0, + "beam_regret_excluded": 4, + "by_origin": {} + }, + "valid_final": { + "n": 2, + "valid_final_rate": 1.0, + "finals": [ + { + "record_id": "fixture_06", + "valid": true, + "steps": 2, + "verifier_calls": 11 + }, + { + "record_id": "fixture_07", + "valid": true, + "steps": 2, + "verifier_calls": 11 + } + ] + }, + "training_metrics": { + "action": 0.6870732307434082, + "value": 0.16490422189235687, + "value_unknown_excluded": 0.0, + "skipped": 0.0, + "rows": 4.0 + } + } + }, + "comparison": { + "best_non_gold_arm": "on_policy", + "beam_regret_improvement": 0.0, + "rank_correlation_degradation": -0.5163977794943222 + }, + "threshold_checks": { + "beam_regret_ok": false, + "rank_ok": true + }, + "verdict": "rejected", + "wall_seconds": 84.94782193800347, + "honesty": "Fixture-scale matched arms over immutable content-addressed state snapshots; evaluation uses held-out seed trajectories only (fail-closed leak guard ran before training); UNKNOWN/unbounded oracle labels are excluded and counted, never coerced. A fixture verdict is wiring/state-source evidence, not a production ship claim.", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "d8f35563d231e3b404b0fb0e213d68aee1b00a9d", + "code_dirty": true, + "components": { + "harness.experiments.slm312_state_sources": "v1", + "harness.experiments.slm299_edit_reachability": "v4" + }, + "stamped_at": "2026-07-24T23:08:24.625582+00:00" + } +} diff --git a/docs/design/iter-slm312-state-sources-20260724.md b/docs/design/iter-slm312-state-sources-20260724.md new file mode 100644 index 000000000..e1170bb48 --- /dev/null +++ b/docs/design/iter-slm312-state-sources-20260724.md @@ -0,0 +1,40 @@ +# SLM-312 (LAR2-04): seedward + on-policy state sources vs near-gold + +**Verdict: `rejected`** (fixture-scale matched arms; not a ship claim) + +## Preregistered (locked before results) + +- arms/weights: `{"gold_only": {"gold_only": 1.0}, "seedward": {"seedward": 1.0}, "on_policy": {"on_policy": 1.0}, "mixed": {"gold_only": 0.34, "seedward": 0.33, "on_policy": 0.33}}` +- per-arm budget: 24 rows; identical model/steps/optimizer/seeds/evaluator +- beam regret improvement >= 0.05 +- rank correlation degradation <= 0.05 +- rule: distribution_gap_closed iff best non-gold arm improves beam regret >= 0.05 AND degrades rank correlation by <= 0.05, else rejected + +## Headline (held-out seed trajectories) + +| arm | rows | rank corr | concordance | Brier | beam regret | valid-final | +| --- | --- | --- | --- | --- | --- | --- | +| gold_only | 17 | -0.2582 | 0.3333 | 0.0340 | 0.0000 | 1.0000 | +| mixed | 24 | 0.2582 | 0.6667 | 0.0068 | 0.0000 | 1.0000 | +| on_policy | 18 | 0.2582 | 0.6667 | 0.0144 | 0.0000 | 1.0000 | +| seedward | 8 | 0.2582 | 0.6667 | 0.0301 | n/a | 1.0000 | + +Best non-gold arm: **on_policy** — beam regret improvement 0.0000, rank correlation degradation -0.5164. + +## State-space coverage by source (full snapshot) + +| source | rows | unique fingerprints | +| --- | --- | --- | +| gold_only | 17 | 17 | +| on_policy | 18 | 18 | +| seedward | 8 | 8 | + +Snapshot duplicate rate 0.140; 37/43 unique states. Manifest `1c5fc80b721b12f4…` at `outputs/slm312/state_snapshot.jsonl`. + +## Coverage + oracle cost + +- UNKNOWN coverage 0.333 (2/6), measurable 4, oracle 1397.1 ms/state + +## Honesty + +Fixture-scale matched arms over immutable content-addressed state snapshots; evaluation uses held-out seed trajectories only (fail-closed leak guard ran before training); UNKNOWN/unbounded oracle labels are excluded and counted, never coerced. A fixture verdict is wiring/state-source evidence, not a production ship claim. diff --git a/scripts/run_slm312_state_sources.py b/scripts/run_slm312_state_sources.py new file mode 100644 index 000000000..fc7386042 --- /dev/null +++ b/scripts/run_slm312_state_sources.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python3 +"""SLM-312 (LAR2-04): matched state-source mixture arms {gold_only, seedward, on_policy, mixed}. + +Question: does closing the gold-corruption vs seed-rollout distribution gap +(training on seedward and/or on-policy valid states, oracle-labeled) improve +repair more than extra offline near-gold training, at equal model / steps / +optimizer / evaluator / seed budget? + +Preregistered (written into the output payload BEFORE any result): + +- arms + mixture weights + acquisition caps: ``PREREGISTERED_ARMS`` and + ``--per-arm-budget`` (locked before training; deviations append-only); +- primary: best non-gold arm must improve beam regret vs ``gold_only`` by + **>= 0.05** (distance units) AND must not degrade value rank correlation + by more than **0.05**; +- verdict ``distribution_gap_closed`` iff both hold, else ``rejected`` — + honestly computed from the measured arms, never narrated. + +Arms share the tiny fixture corpus, model init seed, optimizer budget, batch +order, and evaluator; they differ ONLY in which state source feeds +``state_supervision_loss``. Evaluation runs on HELD-OUT SEED TRAJECTORIES +(random walks from the decode seed over held-out records — never acquired +states): valid-final decode rate, value calibration (rank correlation, +concordance, Brier, per-bin), beam regret vs oracle-best (SLM-308 +machinery), AST-fingerprint duplicate rate + state-space coverage per +source. Leak guards are fail-closed: any train/held-out fingerprint overlap +aborts the run before training. + +Writes ``docs/design/iter-slm312-state-sources-20260724.{json,md}`` and the +immutable snapshot under ``outputs/slm312/``. + +Example: + python -m scripts.run_slm312_state_sources --steps 4 +""" + +from __future__ import annotations + +import argparse +import json +import random +import time +from pathlib import Path + +import torch + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm308_distance_oracle import ( + clear_caches, +) +from slm_training.harnesses.experiments.slm312_state_sources import ( + EXPERIMENT_ID, + PREREGISTERED_ARMS, + SEED_SOURCE, + StateSnapshotV1, + acquire_gold_only, + acquire_on_policy, + acquire_seedward, + assert_no_leakage, + build_arm_rows, + source_coverage, + source_fingerprint, + state_supervision_loss, + drop_leaked_rows, +) +from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + TreeEditSpace, + _is_valid, + parse_statements, + render_statements, +) +from slm_training.versioning import build_version_stamp +from scripts.run_slm308_distance_value import ( + FIXTURE_PROGRAMS, + label_eval_states, + score_arm, +) + +DEFAULT_JSON_OUT = Path("docs/design/iter-slm312-state-sources-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm312-state-sources-20260724.md") +DEFAULT_SNAPSHOT_OUT = Path("outputs/slm312/state_snapshot.jsonl") + +SUITE_ID = "slm312-fixture-v1" +N_TRAIN_RECORDS = 6 # last len(FIXTURE_PROGRAMS) - N_TRAIN_RECORDS are held out + +# Tiny fixture model/decode config — identical across ALL arms and declared in +# the payload; it only bounds fixture wall-clock (the run cap), never differs +# by arm. +FIXTURE_CONFIG_OVERRIDES = { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_search_steps": 8, + "beam_width": 3, + "expand_per_state": 3, +} +ON_POLICY_MAX_STATES_PER_RECORD = 3 + +# PREREGISTERED improvement thresholds (locked before any run; do not edit +# after outcomes are visible — deviations are append-only and exploratory). +PREREGISTERED_THRESHOLDS = { + "beam_regret_improvement_min": 0.05, + "rank_correlation_degradation_max": 0.05, + "comparison": "best non-gold arm (seedward/on_policy/mixed) vs gold_only", + "verdict_rule": ( + "distribution_gap_closed iff best non-gold arm improves beam regret " + ">= 0.05 AND degrades rank correlation by <= 0.05, else rejected" + ), +} + + +def build_records() -> tuple[list[ExampleRecord], list[ExampleRecord]]: + records = [ + ExampleRecord( + id=f"fixture_{i:02d}", + prompt=prompt, + openui=openui, + placeholders=list(placeholders), + split="train" if i < N_TRAIN_RECORDS else "held_out", + ) + for i, (prompt, openui, placeholders) in enumerate(FIXTURE_PROGRAMS) + ] + return records[:N_TRAIN_RECORDS], records[N_TRAIN_RECORDS:] + + +def build_heldout_eval_states( + held_out: list[ExampleRecord], space: TreeEditSpace, *, seed: int = 31337 +) -> list[dict]: + """Held-out seed-trajectory eval states: random walks from the decode + seed over HELD-OUT records only (distinct rng stream from training).""" + states: list[dict] = [] + seed_statements = parse_statements(SEED_SOURCE) + assert seed_statements is not None + for idx, record in enumerate(held_out): + gold = parse_statements(record.openui) + assert gold is not None + inventory = [p if p.startswith(":") else f":{p}" for p in record.placeholders] + rng = random.Random(seed + idx) + # The bare decode seed is the origin of every seed trajectory; it is + # almost always oracle-measurable (small EXACT distance to gold). + states.append( + { + "record_idx": idx, + "origin": "heldout_seed_trajectory", + "witness": None, + "statements": seed_statements, + "target": gold, + "inventory": inventory, + } + ) + # Walk depths 1-2: deeper walks are almost always oracle-UNKNOWN at + # the shallow SLM-308 budgets and only cost eval time. + for k in (1, 2): + current = seed_statements + applied = 0 + for _ in range(k): + step = space.sample_mutation(current, inventory, rng) + if step is None: + break + current, _inverse = step + applied += 1 + if applied == 0: + continue + states.append( + { + "record_idx": idx, + "origin": "heldout_seed_trajectory", + "witness": None, + "statements": current, + "target": gold, + "inventory": inventory, + } + ) + return states + + +def train_arm( + records: list[ExampleRecord], + arm_rows: list, + *, + steps: int, + batch_size: int, + seed: int, +) -> TreeEditDiffusionModel: + """One arm: identical model init / optimizer / batch order; only the + state source differs (loss consumes the arm's snapshot rows).""" + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + seed=seed, + value_label_mode="bounded_distance", + context_backend="scratch", + max_chain=3, + **FIXTURE_CONFIG_OVERRIDES, + ) + model = TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=3e-3) + order = random.Random(0) # identical batch order across arms + record_by_id = {r.id: r for r in records} + model.train() + for _ in range(steps): + batch = [arm_rows[order.randrange(len(arm_rows))] for _ in range(batch_size)] + loss = state_supervision_loss(model, batch, record_by_id) + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +@torch.no_grad() +def valid_final_rate( + model: TreeEditDiffusionModel, held_out: list[ExampleRecord] +) -> dict: + """Decode from the seed per held-out record; fraction whose final beam + state is a valid program.""" + n_valid = 0 + finals: list[dict] = [] + for record in held_out: + inventory = [p if p.startswith(":") else f":{p}" for p in record.placeholders] + prompt = model._format_context( + record.prompt, design_md=record.design_md, slot_contract=inventory + ) + ctx, ctx_pad = model._encode_context([prompt]) + final_text, evidence = model._decode_one(ctx, ctx_pad, inventory) + ok = bool(final_text) and _is_valid(final_text) + n_valid += int(ok) + finals.append( + { + "record_id": record.id, + "valid": ok, + "steps": evidence.get("steps"), + "verifier_calls": evidence.get("verifier_calls"), + } + ) + return { + "n": len(held_out), + "valid_final_rate": n_valid / max(len(held_out), 1), + "finals": finals, + } + + +def build_report( + train_records: list[ExampleRecord], + held_out: list[ExampleRecord], + *, + steps: int, + batch_size: int, + per_arm_budget: int, + seed: int, + snapshot_out: Path, +) -> dict: + space = TreeEditSpace() + clear_caches() + # Tiny CPU fixture: torch thread thrash on tiny ops dominates wall-clock + # (measured 6x slower at 12 threads); pin to a single thread, identical + # for every arm, and declare it in the payload. + torch.set_num_threads(1) + start = time.perf_counter() + + # --- held-out eval states (labeled once, shared across arms) ---------- + eval_states = build_heldout_eval_states(held_out, space) + oracle = label_eval_states(eval_states, space) + held_fingerprints = { + source_fingerprint(render_statements(entry["statements"])) + for entry in eval_states + } + + # --- bootstrap model for on-policy rollouts (gold_only supervision) --- + bootstrap_rows = acquire_gold_only( + train_records, space, parent_checkpoint="none", suite_id=SUITE_ID + ) + bootstrap = train_arm( + train_records, + bootstrap_rows, + steps=steps, + batch_size=batch_size, + seed=seed, + ) + + # --- acquire all sources over TRAIN records only ---------------------- + rows = [] + rows += bootstrap_rows + rows += acquire_seedward( + train_records, space, parent_checkpoint="none", suite_id=SUITE_ID + ) + rows += acquire_on_policy( + bootstrap, + train_records, + space, + parent_checkpoint="bootstrap_gold_only_fixture", + suite_id=SUITE_ID, + max_states_per_record=ON_POLICY_MAX_STATES_PER_RECORD, + ) + + # Drop legitimate cross-record fingerprint collisions (counted in the + # snapshot manifest), then the fail-closed guard verifies the kept set. + rows, dropped_leaks = drop_leaked_rows(rows, held_fingerprints) + assert_no_leakage(rows, held_fingerprints) + + snapshot = StateSnapshotV1( + snapshot_id=f"{EXPERIMENT_ID}-fixture", + config={ + "suite_id": SUITE_ID, + "arms": PREREGISTERED_ARMS, + "per_arm_budget": per_arm_budget, + "steps": steps, + "batch_size": batch_size, + "seed": seed, + "held_out_fingerprints": sorted(held_fingerprints), + "dropped_leak_collisions": dropped_leaks, + }, + rows=rows, + ) + snapshot.write(snapshot_out) + coverage = source_coverage(rows) + + # --- matched arms ------------------------------------------------------- + arms: dict[str, dict] = {} + for arm in sorted(PREREGISTERED_ARMS): + arm_rows = build_arm_rows(rows, arm, per_arm_budget=per_arm_budget, seed=seed) + model = train_arm( + train_records, arm_rows, steps=steps, batch_size=batch_size, seed=seed + ) + metrics = score_arm(model, held_out, eval_states) + arms[arm] = { + "n_rows": len(arm_rows), + "coverage": source_coverage(arm_rows), + "metrics": metrics, + "valid_final": valid_final_rate(model, held_out), + "training_metrics": model.last_training_metrics, + } + + gold = arms["gold_only"]["metrics"] + best_name = None + best_regret_imp: float | None = None + for name in ("seedward", "on_policy", "mixed"): + a = arms[name]["metrics"] + if a["beam_regret_mean"] is None or gold["beam_regret_mean"] is None: + continue + imp = gold["beam_regret_mean"] - a["beam_regret_mean"] + if best_regret_imp is None or imp > best_regret_imp: + best_regret_imp = imp + best_name = name + rank_degradation = None + if ( + best_name is not None + and gold["rank_correlation"] is not None + and arms[best_name]["metrics"]["rank_correlation"] is not None + ): + rank_degradation = ( + gold["rank_correlation"] - arms[best_name]["metrics"]["rank_correlation"] + ) + regret_ok = ( + best_regret_imp is not None + and best_regret_imp >= PREREGISTERED_THRESHOLDS["beam_regret_improvement_min"] + ) + rank_ok = ( + rank_degradation is not None + and rank_degradation + <= PREREGISTERED_THRESHOLDS["rank_correlation_degradation_max"] + ) + verdict = "distribution_gap_closed" if (regret_ok and rank_ok) else "rejected" + + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-312", + "question": ( + "Does closing the gold-corruption vs seed-rollout distribution gap " + "(seedward + on-policy state sources) improve repair more than " + "extra offline near-gold training at equal budget?" + ), + "preregistered_thresholds": PREREGISTERED_THRESHOLDS, + "preregistered_arms": PREREGISTERED_ARMS, + "config": { + "steps": steps, + "batch_size": batch_size, + "per_arm_budget": per_arm_budget, + "seed": seed, + "n_train_records": len(train_records), + "n_held_out_records": len(held_out), + "suite_id": SUITE_ID, + "fixture_config_overrides": FIXTURE_CONFIG_OVERRIDES, + "on_policy_max_states_per_record": ON_POLICY_MAX_STATES_PER_RECORD, + "torch_num_threads": 1, + }, + "snapshot": { + "path": str(snapshot_out), + "manifest": snapshot.manifest(), + "coverage": coverage, + }, + "oracle_cost": oracle, + "arms": arms, + "comparison": { + "best_non_gold_arm": best_name, + "beam_regret_improvement": best_regret_imp, + "rank_correlation_degradation": rank_degradation, + }, + "threshold_checks": {"beam_regret_ok": regret_ok, "rank_ok": rank_ok}, + "verdict": verdict, + "wall_seconds": time.perf_counter() - start, + "honesty": ( + "Fixture-scale matched arms over immutable content-addressed state " + "snapshots; evaluation uses held-out seed trajectories only " + "(fail-closed leak guard ran before training); UNKNOWN/unbounded " + "oracle labels are excluded and counted, never coerced. A fixture " + "verdict is wiring/state-source evidence, not a production ship " + "claim." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm312_state_sources", + "harness.experiments.slm299_edit_reachability", + ) + return payload + + +def render_markdown(payload: dict) -> str: + thr = payload["preregistered_thresholds"] + + def fmt(x: object) -> str: + return "n/a" if x is None else (f"{x:.4f}" if isinstance(x, float) else str(x)) + + lines = [ + "# SLM-312 (LAR2-04): seedward + on-policy state sources vs near-gold", + "", + f"**Verdict: `{payload['verdict']}`** (fixture-scale matched arms; not a ship claim)", + "", + "## Preregistered (locked before results)", + "", + f"- arms/weights: `{json.dumps(payload['preregistered_arms'])}`", + f"- per-arm budget: {payload['config']['per_arm_budget']} rows; " + f"identical model/steps/optimizer/seeds/evaluator", + f"- beam regret improvement >= {thr['beam_regret_improvement_min']}", + f"- rank correlation degradation <= {thr['rank_correlation_degradation_max']}", + f"- rule: {thr['verdict_rule']}", + "", + "## Headline (held-out seed trajectories)", + "", + "| arm | rows | rank corr | concordance | Brier | beam regret | valid-final |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for name, arm in sorted(payload["arms"].items()): + m = arm["metrics"] + lines.append( + f"| {name} | {arm['n_rows']} | {fmt(m['rank_correlation'])} | " + f"{fmt(m['concordance'])} | {fmt(m['brier'])} | " + f"{fmt(m['beam_regret_mean'])} | " + f"{fmt(arm['valid_final']['valid_final_rate'])} |" + ) + cmp_ = payload["comparison"] + lines += [ + "", + f"Best non-gold arm: **{cmp_['best_non_gold_arm']}** — beam regret " + f"improvement {fmt(cmp_['beam_regret_improvement'])}, rank correlation " + f"degradation {fmt(cmp_['rank_correlation_degradation'])}.", + "", + "## State-space coverage by source (full snapshot)", + "", + "| source | rows | unique fingerprints |", + "| --- | --- | --- |", + ] + cov = payload["snapshot"]["coverage"] + for source, row in cov["per_source"].items(): + lines.append(f"| {source} | {row['rows']} | {row['unique_fingerprints']} |") + lines += [ + "", + f"Snapshot duplicate rate {cov['duplicate_rate']:.3f}; " + f"{cov['unique_fingerprints']}/{cov['rows']} unique states. " + f"Manifest `{payload['snapshot']['manifest']['manifest_sha256'][:16]}…` " + f"at `{payload['snapshot']['path']}`.", + "", + "## Coverage + oracle cost", + "", + ] + oracle = payload["oracle_cost"] + lines.append( + f"- UNKNOWN coverage {oracle['unknown_coverage']:.3f} " + f"({oracle['n_unknown']}/{oracle['n_states']}), measurable " + f"{oracle['n_measurable']}, oracle {oracle['ms_per_state']:.1f} ms/state" + ) + lines += [ + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=4) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--per-arm-budget", type=int, default=24) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + parser.add_argument("--snapshot-out", type=Path, default=DEFAULT_SNAPSHOT_OUT) + args = parser.parse_args(argv) + + train_records, held_out = build_records() + payload = build_report( + train_records, + held_out, + steps=args.steps, + batch_size=args.batch_size, + per_arm_budget=args.per_arm_budget, + seed=args.seed, + snapshot_out=args.snapshot_out, + ) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"verdict={payload['verdict']} " + f"best_non_gold={payload['comparison']['best_non_gold_arm']} " + f"regret_imp={payload['comparison']['beam_regret_improvement']} " + f"-> {args.json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm312_state_sources.py b/src/slm_training/harnesses/experiments/slm312_state_sources.py new file mode 100644 index 000000000..b69552582 --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm312_state_sources.py @@ -0,0 +1,1150 @@ +"""SLM-312 (LAR2-04): seedward and on-policy state sources for tree-edit repair. + +Builds immutable, content-addressed training-state snapshots from three +sources over the SLM-305 extended edit space, labels every state with the +SLM-308 bounded distance oracle and an SLM-305 legal supervised edit, and +mixes them into matched ``{gold_only, seedward, on_policy, mixed}`` arms: + +- ``gold_only`` — the existing near-gold corruption chain (``sample_mutation`` + forward noise; the inverse edit is the supervised repair target). +- ``seedward`` — OFFLINE valid intermediates from the minimal decode seed, + reached by greedy oracle-guided edits TOWARD each gold target (SLM-308 + distance strictly decreasing). These cover the seed side of the + distribution that gold corruption never visits. +- ``on_policy`` — immutable snapshots of actual seed-originated beam + trajectories (``_decode_one`` states/proposals telemetry): visited states, + final states, and verifier-failure-cone states, each oracle-labeled. + +Honesty invariants: + +1. Every row carries explicit provenance (parent checkpoint, source commit, + rollout config, prompt/suite hashes, gold-visibility policy). On-policy + ROLLOUTS are gold-blind; oracle LABELS read gold at training time only — + the policy string says exactly which. +2. Leak guard is fail-closed: any AST-fingerprint overlap between a train row + and a held-out evaluation state raises :class:`LeakageError`. +3. Snapshots are content-addressed (per-row sha256 + manifest sha256); a + tampered row or manifest fails :func:`read_snapshot` with + :class:`SnapshotIntegrityError`. +4. UNKNOWN / unbounded oracle labels are excluded from value supervision, + never coerced (same rule as SLM-308). + +This module is torch-free at import time; the loss helper imports torch +lazily so acquisition/guard tests stay cheap. +""" + +from __future__ import annotations + +import hashlib +import json +import random +import subprocess +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Iterable, Sequence + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm299_edit_reachability import ( + _canonical_key, +) +from slm_training.harnesses.experiments.slm308_distance_oracle import ( + DistanceKind, + DistanceLabel, + distance_to_target, + effective_distance, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_ADD, + ACTION_ADD_CONTAINER, + ACTION_BIND_PLACEHOLDER, + ACTION_INSERT_STATEMENT, + ACTION_INSERT_SUBTREE, + ACTION_NAMES, + ACTION_REMOVE, + ACTION_REMOVE_CONTAINER, + ACTION_REPLACE, + ACTION_REPLACE_STATEMENT, + ACTION_REPLACE_SUBTREE, + CONTAINER_RESTS, + Edit, + Statement, + TreeEditSpace, + V05_TEMPLATES, + MAX_SLOTS, + parse_statements, + render_statements, +) + +EXPERIMENT_ID = "slm312-state-sources" +SNAPSHOT_SCHEMA_VERSION = 1 + +SOURCE_GOLD_ONLY = "gold_only" +SOURCE_SEEDWARD = "seedward" +SOURCE_ON_POLICY = "on_policy" +SOURCES = (SOURCE_GOLD_ONLY, SOURCE_SEEDWARD, SOURCE_ON_POLICY) + +# Gold-visibility policies (per row; the on-policy ROLLOUT is gold-blind — +# decode never reads gold — while oracle labels read gold at training time +# only, so the policy string states both phases explicitly). +GOLD_VISIBILITY_ACQUISITION_AND_LABELS = "acquisition_and_labels" +GOLD_VISIBILITY_LABELS_ONLY = "rollout_gold_blind_labels_gold_visible" + +# Oracle budgets for state-source labels (same shallow budgets as SLM-308 +# training-time labels; UNKNOWN is excluded downstream, never coerced). +ORACLE_MAX_DEPTH = 8 +ORACLE_NODE_BUDGET = 8 + +# PREREGISTERED mixture arms (locked before any training; deviations are +# append-only and exploratory). Weights are per-source row-mixture weights. +PREREGISTERED_ARMS: dict[str, dict[str, float]] = { + "gold_only": {SOURCE_GOLD_ONLY: 1.0}, + "seedward": {SOURCE_SEEDWARD: 1.0}, + "on_policy": {SOURCE_ON_POLICY: 1.0}, + "mixed": {SOURCE_GOLD_ONLY: 0.34, SOURCE_SEEDWARD: 0.33, SOURCE_ON_POLICY: 0.33}, +} + +SEED_SOURCE = 'root = Stack([], "column")' + + +class LeakageError(RuntimeError): + """A train row overlaps a held-out evaluation state (fail-closed).""" + + +class SnapshotIntegrityError(RuntimeError): + """A content-addressed snapshot row or manifest failed verification.""" + + +# --------------------------------------------------------------------------- # +# Hashing helpers +# --------------------------------------------------------------------------- # + + +def _canonical(obj: Any) -> str: + return json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str) + + +def _digest(obj: Any) -> str: + return hashlib.sha256(_canonical(obj).encode("utf-8")).hexdigest() + + +def ast_fingerprint(statements: list[Statement]) -> str: + """Alpha-invariant AST fingerprint (SLM-299 canonical key), sha256.""" + return hashlib.sha256(_canonical_key(statements).encode("utf-8")).hexdigest() + + +def source_fingerprint(source_text: str) -> str: + """Fingerprint of a rendered program; raises when it does not parse.""" + statements = parse_statements(source_text) + if statements is None: + raise ValueError(f"statements do not parse: {source_text!r}") + return ast_fingerprint(statements) + + +def _git_commit() -> str: + try: + out = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + timeout=10, + ) + return out.stdout.strip() + except Exception: # noqa: BLE001 + return "unknown" + + +def _text_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16] + + +# --------------------------------------------------------------------------- # +# Provenance + row schema +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class StateProvenance: + """Where one training state came from, and who was allowed to see gold.""" + + parent_checkpoint: str + source_commit: str + rollout_config: dict[str, Any] + prompt_hash: str + suite_hash: str + gold_visibility_policy: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "StateProvenance": + return cls( + parent_checkpoint=str(data["parent_checkpoint"]), + source_commit=str(data["source_commit"]), + rollout_config=dict(data.get("rollout_config") or {}), + prompt_hash=str(data["prompt_hash"]), + suite_hash=str(data["suite_hash"]), + gold_visibility_policy=str(data["gold_visibility_policy"]), + ) + + +def edit_to_dict(edit: Edit) -> dict[str, Any]: + return { + "action": edit.action, + "action_name": ACTION_NAMES[edit.action], + "stmt": edit.stmt, + "comp": edit.comp, + "slot": edit.slot, + "target": edit.target, + "payload": edit.payload, + } + + +def edit_from_dict(data: dict[str, Any]) -> Edit: + return Edit( + int(data["action"]), + stmt=int(data.get("stmt", 0)), + comp=int(data.get("comp", 0)), + slot=int(data.get("slot", 0)), + target=int(data.get("target", 0)), + payload=int(data.get("payload", 0)), + ) + + +@dataclass(frozen=True) +class StateSourceRow: + """One labeled training state from one source, immutable + addressed.""" + + record_id: str + split: str # "train" | "held_out" (held_out rows are eval-only) + source: str # one of SOURCES + kind: str # near_gold | seedward_step | visited | final | verifier_failure_cone + statements_source: str + inventory: tuple[str, ...] + supervised_edit: dict[str, Any] | None # SLM-305 legal edit toward gold + child_source: str | None # apply(supervised_edit); strictly closer when set + distance_label: dict[str, Any] # DistanceLabel.to_dict() + value_target: float | None # None => exclude from value loss + provenance: StateProvenance + schema_version: int = SNAPSHOT_SCHEMA_VERSION + + def content(self) -> dict[str, Any]: + data = asdict(self) + data["inventory"] = list(self.inventory) + data["provenance"] = self.provenance.to_dict() + return data + + def sha256(self) -> str: + return _digest(self.content()) + + def fingerprint(self) -> str: + return source_fingerprint(self.statements_source) + + def to_dict(self) -> dict[str, Any]: + data = self.content() + data["sha256"] = self.sha256() + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "StateSourceRow": + row = cls( + record_id=str(data["record_id"]), + split=str(data["split"]), + source=str(data["source"]), + kind=str(data["kind"]), + statements_source=str(data["statements_source"]), + inventory=tuple(str(x) for x in data.get("inventory") or ()), + supervised_edit=( + dict(data["supervised_edit"]) + if data.get("supervised_edit") is not None + else None + ), + child_source=data.get("child_source"), + distance_label=dict(data["distance_label"]), + value_target=( + float(data["value_target"]) + if data.get("value_target") is not None + else None + ), + provenance=StateProvenance.from_dict(data["provenance"]), + schema_version=int(data.get("schema_version", SNAPSHOT_SCHEMA_VERSION)), + ) + expected = data.get("sha256") + if expected is not None and expected != row.sha256(): + raise SnapshotIntegrityError( + f"row {row.record_id}/{row.kind} sha256 mismatch: " + f"stored {expected}, computed {row.sha256()}" + ) + return row + + +def make_provenance( + record: ExampleRecord, + *, + parent_checkpoint: str, + rollout_config: dict[str, Any], + suite_id: str, + gold_visibility_policy: str, + source_commit: str | None = None, +) -> StateProvenance: + return StateProvenance( + parent_checkpoint=parent_checkpoint, + source_commit=source_commit if source_commit is not None else _git_commit(), + rollout_config=dict(rollout_config), + prompt_hash=_text_hash(record.prompt), + suite_hash=_text_hash(suite_id), + gold_visibility_policy=gold_visibility_policy, + ) + + +def _label_row( + *, + record: ExampleRecord, + split: str, + source: str, + kind: str, + statements: list[Statement], + target: list[Statement], + inventory: Sequence[str], + supervised_edit: Edit | None, + space: TreeEditSpace, + provenance: StateProvenance, + witness: int | None, + child_source: str | None = None, + verify_edit: bool = True, +) -> StateSourceRow: + """Oracle-label one state. + + With ``verify_edit`` (default) the supervised edit must be SLM-305 legal + on this state (fail-closed ``apply``); the resulting child is the + pairwise-progress target. Gold-corruption rows instead pass the actual + chain parent as ``child_source`` with ``verify_edit=False`` — the + historical inverse edit is a supervision target in index space and is + never re-applied (same semantics as ``training_loss``). + """ + if supervised_edit is not None and verify_edit: + child = space.apply(statements, supervised_edit, list(inventory)) + if child is None: + raise ValueError( + f"supervised edit {edit_to_dict(supervised_edit)} is not legal " + f"for {render_statements(statements)!r} (fail-closed apply)" + ) + child_source = render_statements(child) + label = distance_to_target( + statements, + target, + space=space, + inventory=inventory, + max_depth=ORACLE_MAX_DEPTH, + node_budget=ORACLE_NODE_BUDGET, + upper_bound_witness=witness, + ) + return StateSourceRow( + record_id=record.id, + split=split, + source=source, + kind=kind, + statements_source=render_statements(statements), + inventory=tuple(inventory), + supervised_edit=( + edit_to_dict(supervised_edit) if supervised_edit is not None else None + ), + child_source=child_source, + distance_label=label.to_dict(), + value_target=label.value_target(ORACLE_MAX_DEPTH), + provenance=provenance, + ) + + +# --------------------------------------------------------------------------- # +# Leak guard (fail-closed) +# --------------------------------------------------------------------------- # + + +def drop_leaked_rows( + rows: Iterable[StateSourceRow], + held_out_fingerprints: Iterable[str], +) -> tuple[list[StateSourceRow], list[dict[str, str]]]: + """Remove TRAIN rows whose AST fingerprint collides with a held-out eval + state, returning (kept, dropped-log). Legitimate cross-record collisions + (alpha-invariant canonical keys make e.g. a one-leaf near-gold state of a + train record and a seed-walk state of a held-out record identical) are + dropped and counted — never silently kept. :func:`assert_no_leakage` + remains the fail-closed verification on the kept set.""" + held = set(held_out_fingerprints) + kept: list[StateSourceRow] = [] + dropped: list[dict[str, str]] = [] + for row in rows: + if row.split == "train" and row.fingerprint() in held: + dropped.append( + { + "record_id": row.record_id, + "source": row.source, + "kind": row.kind, + "fingerprint": row.fingerprint(), + "reason": "held_out_fingerprint_collision", + } + ) + continue + kept.append(row) + return kept, dropped + + +def assert_no_leakage( + rows: Iterable[StateSourceRow], + held_out_fingerprints: Iterable[str], +) -> None: + """Raise LeakageError when any TRAIN row's AST fingerprint appears in the + held-out evaluation fingerprint set. Held-out rows are eval-only and are + never train supervision, so only ``split == "train"`` rows are checked.""" + held = set(held_out_fingerprints) + if not held: + return + overlaps: list[dict[str, str]] = [] + for row in rows: + if row.split != "train": + continue + fp = row.fingerprint() + if fp in held: + overlaps.append( + { + "record_id": row.record_id, + "source": row.source, + "kind": row.kind, + "fingerprint": fp, + } + ) + if overlaps: + raise LeakageError( + f"{len(overlaps)} train row(s) overlap held-out eval states: " + + json.dumps(overlaps[:5], sort_keys=True) + ) + + +# --------------------------------------------------------------------------- # +# Child-edit enumeration (mirrors slm299's extended mode but keeps the Edit) +# --------------------------------------------------------------------------- # + + +def enumerate_child_edits( + space: TreeEditSpace, + statements: list[Statement], + inventory: Sequence[str], + *, + visited: set[str] | None = None, +) -> list[tuple[list[Statement], Edit]]: + """All one-edit successors under the deployed extended action set, + applied through ``TreeEditSpace.apply`` (fail-closed validity), keeping + the :class:`Edit` so it can be used as a supervision target. Mirrors + ``slm299_edit_reachability._enumerate_children(mode="extended")``.""" + from slm_training.models.tree_edit_diffusion import ( + CONTAINER_COMPONENTS, + LEAF_COMPONENTS, + ) + + children: list[tuple[list[Statement], Edit]] = [] + n_slots = min(len(inventory), MAX_SLOTS) + leaf_comp_idxs = [i for i, c in enumerate(space.components) if c in LEAF_COMPONENTS] + container_comp_idxs = [ + i for i, c in enumerate(space.components) if c in CONTAINER_COMPONENTS + ] + pre = None + if visited is not None: + pre = lambda working: _canonical_key(working) not in visited # noqa: E731 + by_name = {s.name: s for s in statements} + + def _replace_subtree_ok(stmt: Statement) -> bool: + if not stmt.has_list or len(stmt.children) != 1: + return False + leaf = by_name.get(stmt.children[0]) + return leaf is not None and not leaf.has_list + + for stmt_idx in range(len(statements)): + stmt = statements[stmt_idx] + for comp_idx in range(len(space.components)): + edit = Edit(ACTION_REPLACE, stmt_idx, comp_idx) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + for slot_idx in range(n_slots): + edit = Edit(ACTION_ADD, stmt_idx, comp_idx, slot_idx) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + if stmt.has_list: + for comp_idx in container_comp_idxs: + for rest_idx in range(len(CONTAINER_RESTS)): + edit = Edit( + ACTION_ADD_CONTAINER, stmt_idx, comp_idx, target=rest_idx + ) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + for comp_idx in container_comp_idxs: + for slot_idx in range(n_slots): + for payload in leaf_comp_idxs: + for rest_idx in range(len(CONTAINER_RESTS)): + edit = Edit( + ACTION_INSERT_SUBTREE, + stmt_idx, + comp_idx, + slot_idx, + target=rest_idx, + payload=payload, + ) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + if _replace_subtree_ok(stmt): + for slot_idx in range(n_slots): + for payload in leaf_comp_idxs: + edit = Edit( + ACTION_REPLACE_SUBTREE, + stmt_idx, + slot=slot_idx, + payload=payload, + ) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + else: + for slot_idx in range(n_slots): + edit = Edit(ACTION_BIND_PLACEHOLDER, stmt_idx, slot=slot_idx) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + edit = Edit(ACTION_REMOVE_CONTAINER, stmt_idx) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + edit = Edit(ACTION_REMOVE, stmt_idx) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + for payload in range(len(V05_TEMPLATES)): + edit = Edit(ACTION_REPLACE_STATEMENT, stmt_idx, payload=payload) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + for payload in range(len(V05_TEMPLATES)): + edit = Edit(ACTION_INSERT_STATEMENT, payload=payload) + nxt = space.apply(statements, edit, list(inventory), pre) + if nxt is not None: + children.append((nxt, edit)) + return children + + +def oracle_best_child( + statements: list[Statement], + target: list[Statement], + space: TreeEditSpace, + inventory: Sequence[str], +) -> tuple[Edit, list[Statement], DistanceLabel] | None: + """Strictly-improving child with the smallest oracle distance, or None. + + Deterministic: children are enumerated in fixed order and ties keep the + first. Returns the edit (a legal SLM-305 action by construction), the + child statements, and the child's distance label. + """ + current = distance_to_target( + statements, + target, + space=space, + inventory=inventory, + max_depth=ORACLE_MAX_DEPTH, + node_budget=ORACLE_NODE_BUDGET, + ) + d_current = effective_distance(current) + best: tuple[float, Edit, list[Statement], DistanceLabel] | None = None + for child, edit in enumerate_child_edits(space, statements, inventory): + label = distance_to_target( + child, + target, + space=space, + inventory=inventory, + max_depth=ORACLE_MAX_DEPTH, + node_budget=ORACLE_NODE_BUDGET, + ) + d = effective_distance(label) + if d is None: + continue + if d_current is not None and d >= d_current: + continue + if best is None or d < best[0]: + best = (d, edit, child, label) + if best is None: + return None + return best[1], best[2], best[3] + + +# --------------------------------------------------------------------------- # +# Acquisition +# --------------------------------------------------------------------------- # + + +def _inventory_of(record: ExampleRecord) -> list[str]: + return [p if p.startswith(":") else f":{p}" for p in (record.placeholders or ())] + + +def acquire_gold_only( + records: Sequence[ExampleRecord], + space: TreeEditSpace, + *, + split: str = "train", + k_max: int = 3, + seed: int = 20260724, + parent_checkpoint: str = "none", + suite_id: str = "slm312-fixture", +) -> list[StateSourceRow]: + """Existing near-gold corruption chain: mutate gold 1..k edits; the + inverse edit is the supervised repair target (current training path).""" + rows: list[StateSourceRow] = [] + for idx, record in enumerate(records): + gold = parse_statements(record.openui or "") + if gold is None: + continue + inventory = _inventory_of(record) + provenance = make_provenance( + record, + parent_checkpoint=parent_checkpoint, + rollout_config={"acquisition": "sample_mutation", "k_max": k_max}, + suite_id=suite_id, + gold_visibility_policy=GOLD_VISIBILITY_ACQUISITION_AND_LABELS, + ) + rng = random.Random(seed + idx) + for k in (1, 2, 3): + if k > k_max: + break + current = gold + prev = None + inverse: Edit | None = None + applied = 0 + for _ in range(k): + step = space.sample_mutation(current, inventory, rng) + if step is None: + break + prev = current + current, inverse = step + applied += 1 + if inverse is None: + continue + rows.append( + _label_row( + record=record, + split=split, + source=SOURCE_GOLD_ONLY, + kind="near_gold", + statements=current, + target=gold, + inventory=inventory, + supervised_edit=inverse, + space=space, + provenance=provenance, + witness=applied, + # Pairwise child = the actual chain parent (one inverse + # step closer to gold), mirroring training_loss; the + # inverse edit itself is an index-space CE target and is + # never re-applied there, so it is not re-applied here. + child_source=render_statements(prev) if prev is not None else None, + verify_edit=False, + ) + ) + return rows + + +def acquire_seedward( + records: Sequence[ExampleRecord], + space: TreeEditSpace, + *, + split: str = "train", + seed_source: str = SEED_SOURCE, + max_edits: int = 4, + parent_checkpoint: str = "none", + suite_id: str = "slm312-fixture", +) -> list[StateSourceRow]: + """OFFLINE seedward source: from the minimal decode seed, walk greedy + oracle-guided edits TOWARD each gold target, emitting every valid + intermediate (the seed side of the distribution). Every step is a legal + SLM-305 action and strictly distance-decreasing by construction.""" + rows: list[StateSourceRow] = [] + seed_statements = parse_statements(seed_source) + if seed_statements is None: + raise ValueError(f"seed source does not parse: {seed_source!r}") + for record in records: + gold = parse_statements(record.openui or "") + if gold is None: + continue + inventory = _inventory_of(record) + provenance = make_provenance( + record, + parent_checkpoint=parent_checkpoint, + rollout_config={ + "acquisition": "oracle_guided_greedy", + "seed_source": seed_source, + "max_edits": max_edits, + }, + suite_id=suite_id, + gold_visibility_policy=GOLD_VISIBILITY_ACQUISITION_AND_LABELS, + ) + current = seed_statements + chain: list[tuple[list[Statement], Edit | None]] = [] # (state, edit to next) + for _ in range(max_edits): + found = oracle_best_child(current, gold, space, inventory) + if found is None: + break + edit, child, _label = found + chain.append((child, edit)) + current = child + # Row i's supervised edit is the edit taken NEXT (toward gold); the + # final chain state gets its own oracle-best continuation, or STOP. + for i, (state, _took) in enumerate(chain): + if i + 1 < len(chain): + supervised: Edit | None = chain[i + 1][1] + else: + nxt = oracle_best_child(state, gold, space, inventory) + supervised = nxt[0] if nxt is not None else None + rows.append( + _label_row( + record=record, + split=split, + source=SOURCE_SEEDWARD, + kind="seedward_step", + statements=state, + target=gold, + inventory=inventory, + supervised_edit=supervised, + space=space, + provenance=provenance, + # Greedy seed-side steps do not prove a path to gold, so + # there is no honest upper-bound witness here. + witness=None, + ) + ) + return rows + + +def acquire_on_policy( + model: Any, + records: Sequence[ExampleRecord], + space: TreeEditSpace, + *, + split: str = "train", + parent_checkpoint: str, + suite_id: str = "slm312-fixture", + max_states_per_record: int = 8, +) -> list[StateSourceRow]: + """Immutable snapshots from ACTUAL seed-originated beam trajectories. + + Runs ``model._decode_one`` per record (gold-blind rollout), then labels + the visited/final states with the training-time oracle. Verifier-failure + cone states (states with at least one dead proposal) are tagged so the + failure neighborhoods are represented and are prioritized under + ``max_states_per_record`` (oracle-guided relabeling is search-backed, so + the per-record cap keeps acquisition cost bounded at fixture scale). + Rows are deduped by fingerprint, keeping the richest kind tag. + """ + rows: list[StateSourceRow] = [] + seen: set[str] = set() + rollout_config = { + "beam_width": model.config.beam_width, + "expand_per_state": model.config.expand_per_state, + "max_search_steps": model.config.max_search_steps, + "stop_slot_accounting": model.config.stop_slot_accounting, + } + for record in records: + gold = parse_statements(record.openui or "") + if gold is None: + continue + inventory = _inventory_of(record) + provenance = make_provenance( + record, + parent_checkpoint=parent_checkpoint, + rollout_config=rollout_config, + suite_id=suite_id, + gold_visibility_policy=GOLD_VISIBILITY_LABELS_ONLY, + ) + prompt = model._format_context( + record.prompt, design_md=record.design_md, slot_contract=inventory + ) + ctx, ctx_pad = model._encode_context([prompt]) + final_text, evidence = model._decode_one(ctx, ctx_pad, inventory) + if not final_text: + continue + # States with at least one dead (verifier-rejected) proposal. + failure_states = { + (p["step"], p["beam_row"]) + for p in evidence.get("proposals", []) + if not p.get("applicable", False) + } + candidates: list[tuple[str, str]] = [] # (kind, source) + for entry in evidence.get("states", []): + kind = ( + "verifier_failure_cone" + if (entry["step"], entry["beam_row"]) in failure_states + else "visited" + ) + candidates.append((kind, entry["source"])) + candidates.append(("final", final_text)) + # Prioritize failure-cone + final states under the per-record cap. + priority = {"verifier_failure_cone": 0, "final": 1, "visited": 2} + candidates.sort(key=lambda item: priority.get(item[0], 3)) + per_record = 0 + for kind, source in candidates: + if per_record >= max_states_per_record: + break + statements = parse_statements(source) + if statements is None: + continue + fp = ast_fingerprint(statements) + if fp in seen: + continue + seen.add(fp) + per_record += 1 + found = oracle_best_child(statements, gold, space, inventory) + edit = found[0] if found is not None else None + rows.append( + _label_row( + record=record, + split=split, + source=SOURCE_ON_POLICY, + kind=kind, + statements=statements, + target=gold, + inventory=inventory, + supervised_edit=edit, + space=space, + provenance=provenance, + witness=None, + ) + ) + return rows + + +# --------------------------------------------------------------------------- # +# Immutable content-addressed snapshots +# --------------------------------------------------------------------------- # + + +@dataclass +class StateSnapshotV1: + """A frozen set of labeled state rows with a content-address manifest.""" + + snapshot_id: str + config: dict[str, Any] + rows: list[StateSourceRow] = field(default_factory=list) + + def manifest(self) -> dict[str, Any]: + row_hashes = [r.sha256() for r in self.rows] + return { + "snapshot_schema": f"slm312-state-snapshot/v{SNAPSHOT_SCHEMA_VERSION}", + "snapshot_id": self.snapshot_id, + "config": self.config, + "n_rows": len(self.rows), + "row_sha256": row_hashes, + "manifest_sha256": _digest( + { + "snapshot_id": self.snapshot_id, + "config": self.config, + "row_sha256": row_hashes, + } + ), + } + + def write(self, path: Path) -> Path: + """Write rows as JSONL plus a ``.manifest.json`` sidecar.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + for row in self.rows: + fh.write(json.dumps(row.to_dict(), sort_keys=True) + "\n") + manifest_path = path.with_suffix(path.suffix + ".manifest.json") + manifest_path.write_text( + json.dumps(self.manifest(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return path + + +def read_snapshot(path: Path) -> StateSnapshotV1: + """Load a snapshot, verifying every row hash and the manifest (tamper + detection is fail-closed: any mismatch raises SnapshotIntegrityError).""" + manifest_path = path.with_suffix(path.suffix + ".manifest.json") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + rows: list[StateSourceRow] = [] + with path.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line: + rows.append(StateSourceRow.from_dict(json.loads(line))) + if [r.sha256() for r in rows] != list(manifest.get("row_sha256") or []): + raise SnapshotIntegrityError( + f"row hash sequence mismatch against manifest {manifest_path}" + ) + snapshot = StateSnapshotV1( + snapshot_id=str(manifest["snapshot_id"]), + config=dict(manifest.get("config") or {}), + rows=rows, + ) + if snapshot.manifest()["manifest_sha256"] != manifest.get("manifest_sha256"): + raise SnapshotIntegrityError(f"manifest sha256 mismatch for {manifest_path}") + return snapshot + + +# --------------------------------------------------------------------------- # +# Mixture arms (predeclared weights + caps BEFORE any training) +# --------------------------------------------------------------------------- # + + +def build_arm_rows( + rows: Sequence[StateSourceRow], + arm: str, + *, + per_arm_budget: int, + seed: int = 0, +) -> list[StateSourceRow]: + """Deterministic per-source sampling under the predeclared arm weights. + + Every arm draws at most ``per_arm_budget`` rows in total; the per-source + quota is ``round(weight * budget)`` (at least 1 for a positive weight), + sampled without replacement with a per-arm seed so arms share budgets but + not rows. + """ + if arm not in PREREGISTERED_ARMS: + raise ValueError( + f"unknown arm {arm!r} (predeclared: {sorted(PREREGISTERED_ARMS)})" + ) + weights = PREREGISTERED_ARMS[arm] + train_rows = [r for r in rows if r.split == "train"] + rng = random.Random(seed) + selected: list[StateSourceRow] = [] + for source in sorted(weights): + weight = weights[source] + if weight <= 0: + continue + quota = max(1, round(weight * per_arm_budget)) + pool = [r for r in train_rows if r.source == source] + # Dedupe by fingerprint within the source pool. + seen: set[str] = set() + unique: list[StateSourceRow] = [] + for row in pool: + fp = row.fingerprint() + if fp in seen: + continue + seen.add(fp) + unique.append(row) + n = min(quota, len(unique)) + selected.extend(rng.sample(unique, n) if n < len(unique) else unique) + return selected[:per_arm_budget] + + +def source_coverage(rows: Sequence[StateSourceRow]) -> dict[str, Any]: + """Per-source state-space coverage + AST-fingerprint duplicate rate.""" + per_source: dict[str, set[str]] = {} + counts: dict[str, int] = {} + for row in rows: + per_source.setdefault(row.source, set()).add(row.fingerprint()) + counts[row.source] = counts.get(row.source, 0) + 1 + total = len(rows) + unique_total = len({r.fingerprint() for r in rows}) + return { + "rows": total, + "unique_fingerprints": unique_total, + "duplicate_rate": (1.0 - unique_total / total) if total else 0.0, + "per_source": { + source: { + "rows": counts.get(source, 0), + "unique_fingerprints": len(fps), + } + for source, fps in sorted(per_source.items()) + }, + } + + +# --------------------------------------------------------------------------- # +# Training loss over acquired rows (mirrors training_loss mechanics) +# --------------------------------------------------------------------------- # + + +def state_supervision_loss( + model: Any, + rows: Sequence[StateSourceRow], + record_by_id: dict[str, ExampleRecord], +) -> Any: + """Edit-prediction + bounded-distance value loss over acquired state rows. + + Same loss terms as ``TreeEditDiffusionModel.training_loss`` (policy CE on + the supervised edit, value MSE with UNKNOWN masked out, pairwise progress + ranking toward the supervised child) but the states come from the arm's + snapshot rows instead of an internally sampled gold-corruption chain — + so the four arms share identical code and differ ONLY in state source. + Rows without a supervised edit (e.g. a state already at/oracle-closest to + gold) are supervised as STOP when the oracle proves distance 0, else they + carry value supervision only. + """ + import torch + import torch.nn.functional as F + + from slm_training.models.tree_edit_diffusion import ( + ACTION_STOP, + MAX_STMTS, + pairwise_progress_loss, + ) + + prompts: list[str] = [] + states: list[str] = [] + targets: list[Edit] = [] + values: list[float] = [] + value_mask: list[bool] = [] + pair_rows: list[tuple[int, str]] = [] + skipped = 0 + n_unknown_excluded = 0 + for row in rows: + record = record_by_id.get(row.record_id) + if record is None: + skipped += 1 + continue + statements = parse_statements(row.statements_source) + if statements is None: + skipped += 1 + continue + inventory = list(row.inventory) + prompt = model._format_context( + record.prompt, design_md=record.design_md, slot_contract=inventory + ) + prompts.append(prompt) + states.append(row.statements_source) + if row.supervised_edit is not None: + targets.append(edit_from_dict(row.supervised_edit)) + else: + targets.append(Edit(ACTION_STOP)) + if row.value_target is None: + values.append(0.0) + value_mask.append(False) + n_unknown_excluded += 1 + else: + values.append(float(row.value_target)) + value_mask.append(True) + if row.child_source is not None: + # Pairwise progress: the supervised child is strictly closer by + # construction (gold inverse edit or oracle-certified improving + # edit); the oracle labels on the row prove comparability. + d_parent = effective_distance( + DistanceLabel( + kind=DistanceKind(row.distance_label["kind"]), + reason=str(row.distance_label["reason"]), + distance=row.distance_label.get("distance"), + lo=row.distance_label.get("lo"), + hi=row.distance_label.get("hi"), + ) + ) + if d_parent is not None and d_parent > 0: + pair_rows.append((len(states) - 1, row.child_source)) + if not states: + return torch.zeros((), device=model.device_name, requires_grad=True) + n_main = len(states) + pair_child_rows = list(range(n_main, n_main + len(pair_rows))) + states.extend(child for _, child in pair_rows) + ctx, ctx_pad = model._encode_context(prompts) + if pair_rows: + ctx_rows = [row for row, _ in pair_rows] + ctx = torch.cat([ctx, ctx[ctx_rows]], dim=0) + ctx_pad = torch.cat([ctx_pad, ctx_pad[ctx_rows]], dim=0) + out = model.policy(model._state_batch(states), model.tokenizer.pad_id, ctx, ctx_pad) + device = model.device_name + action_t = torch.tensor([e.action for e in targets], device=device) + loss = F.cross_entropy(out["action"][:n_main], action_t) + losses = {"action": float(loss.detach().cpu())} + stmt_rows = [i for i, e in enumerate(targets) if e.action != ACTION_STOP] + if stmt_rows: + idx = torch.tensor(stmt_rows, device=device) + stmt_t = torch.tensor( + [min(targets[i].stmt, MAX_STMTS - 1) for i in stmt_rows], device=device + ) + stmt_loss = F.cross_entropy(out["stmt"][idx], stmt_t) + loss = loss + stmt_loss + losses["stmt"] = float(stmt_loss.detach().cpu()) + comp_rows = [ + i + for i, e in enumerate(targets) + if e.action + in {ACTION_REPLACE, ACTION_ADD, ACTION_ADD_CONTAINER, ACTION_INSERT_SUBTREE} + ] + if comp_rows: + idx = torch.tensor(comp_rows, device=device) + comp_t = torch.tensor([targets[i].comp for i in comp_rows], device=device) + comp_loss = F.cross_entropy(out["comp"][idx], comp_t) + loss = loss + comp_loss + losses["comp"] = float(comp_loss.detach().cpu()) + slot_rows = [ + i + for i, e in enumerate(targets) + if e.action + in { + ACTION_ADD, + ACTION_INSERT_SUBTREE, + ACTION_REPLACE_SUBTREE, + ACTION_BIND_PLACEHOLDER, + } + ] + if slot_rows: + idx = torch.tensor(slot_rows, device=device) + slot_t = torch.tensor( + [min(targets[i].slot, MAX_SLOTS - 1) for i in slot_rows], device=device + ) + slot_loss = F.cross_entropy(out["slot"][idx], slot_t) + loss = loss + slot_loss + losses["slot"] = float(slot_loss.detach().cpu()) + value_t = torch.tensor(values, device=device, dtype=out["value"].dtype) + mask_t = torch.tensor(value_mask, device=device, dtype=torch.bool) + if bool(mask_t.any()): + value_loss = F.mse_loss(out["value"][:n_main][mask_t], value_t[mask_t]) + loss = loss + value_loss + losses["value"] = float(value_loss.detach().cpu()) + if pair_rows: + pair_loss = pairwise_progress_loss( + out["value"][torch.tensor([row for row, _ in pair_rows], device=device)], + out["value"][torch.tensor(pair_child_rows, device=device)], + margin=model.config.pairwise_progress_margin, + ) + loss = loss + pair_loss + losses["pairwise_progress"] = float(pair_loss.detach().cpu()) + losses["value_unknown_excluded"] = float(n_unknown_excluded) + losses["skipped"] = float(skipped) + losses["rows"] = float(n_main) + model.last_training_metrics = losses + return loss + + +__all__ = [ + "EXPERIMENT_ID", + "GOLD_VISIBILITY_ACQUISITION_AND_LABELS", + "GOLD_VISIBILITY_LABELS_ONLY", + "LeakageError", + "PREREGISTERED_ARMS", + "SEED_SOURCE", + "SOURCES", + "SOURCE_GOLD_ONLY", + "SOURCE_ON_POLICY", + "SOURCE_SEEDWARD", + "SNAPSHOT_SCHEMA_VERSION", + "SnapshotIntegrityError", + "StateProvenance", + "StateSnapshotV1", + "StateSourceRow", + "acquire_gold_only", + "acquire_on_policy", + "acquire_seedward", + "assert_no_leakage", + "ast_fingerprint", + "build_arm_rows", + "drop_leaked_rows", + "edit_from_dict", + "edit_to_dict", + "enumerate_child_edits", + "make_provenance", + "oracle_best_child", + "read_snapshot", + "source_coverage", + "source_fingerprint", + "state_supervision_loss", +] diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 015d3f848..1ec0d23fb 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -7841,6 +7841,24 @@ "note": "initial registration; SLM-310 LAR2-03 action alignment: reason-coded proposal telemetry, train-target vs decode-demand distribution audit, declared inverse-action corruption sampler, legacy/corrected STOP-slot accounting arms, matched 2x2 fixture experiment with preregistered thresholds" } ] + }, + "harness.experiments.slm312_state_sources": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm312_state_sources.py", + "scripts/run_slm312_state_sources.py", + "tests/test_harnesses/experiments/test_slm312_state_sources.py", + "docs/design/iter-slm312-state-sources-20260724.json", + "docs/design/iter-slm312-state-sources-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-312 LAR2-04 state sources: offline seedward oracle-guided valid intermediates + immutable content-addressed on-policy beam-trajectory snapshots labeled with the SLM-308 distance oracle and SLM-305 legal edits, fail-closed AST-fingerprint leak guards, matched {gold_only, seedward, on_policy, mixed} mixture arms with preregistered thresholds" + } + ] } } } diff --git a/tests/test_harnesses/experiments/test_slm312_state_sources.py b/tests/test_harnesses/experiments/test_slm312_state_sources.py new file mode 100644 index 000000000..9d61a1727 --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm312_state_sources.py @@ -0,0 +1,310 @@ +"""SLM-312 (LAR2-04): state-source acquisition, snapshots, leak guards, arms.""" + +from __future__ import annotations + +import json + +import pytest + +# Tiny CPU fixture models: torch thread thrash dominates otherwise. +import torch + +torch.set_num_threads(1) + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm308_distance_oracle import ( + DistanceKind, + clear_caches, + distance_to_target, + effective_distance, +) +from slm_training.harnesses.experiments.slm312_state_sources import ( + GOLD_VISIBILITY_ACQUISITION_AND_LABELS, + GOLD_VISIBILITY_LABELS_ONLY, + PREREGISTERED_ARMS, + SOURCE_SEEDWARD, + LeakageError, + SnapshotIntegrityError, + StateSnapshotV1, + acquire_gold_only, + acquire_seedward, + assert_no_leakage, + build_arm_rows, + drop_leaked_rows, + edit_from_dict, + oracle_best_child, + read_snapshot, + source_fingerprint, + state_supervision_loss, +) +from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + TreeEditSpace, + parse_statements, + render_statements, +) + +GOLD = 'root = Stack([t, b], "column")\nt = TextContent(":hero.title")\nb = Button(":cta.label")' +SEED = 'root = Stack([], "column")' + +# Tiny fixture model config (same overrides as the CLI) to keep tests fast. +_SMALL_CONFIG = { + "context_backend": "scratch", + "seed": 0, + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3, +} + + +@pytest.fixture() +def space(): + return TreeEditSpace() + + +@pytest.fixture(autouse=True) +def _clean_oracle_caches(): + clear_caches() + yield + clear_caches() + + +@pytest.fixture() +def records(): + return [ + ExampleRecord( + id="train_0", + prompt="hero with cta", + openui=GOLD, + placeholders=[":hero.title", ":cta.label"], + split="train", + ), + ExampleRecord( + id="train_1", + prompt="form with input", + openui='root = Form([i, s], "column")\ni = TextInput(":form.email")\ns = Button(":form.submit")', + placeholders=[":form.email", ":form.submit"], + split="train", + ), + ] + + +# --- provenance is explicit per row ------------------------------------------ + + +def test_provenance_explicit_per_row(space, records) -> None: + rows = acquire_gold_only(records, space, parent_checkpoint="ckpt-abc") + assert rows, "expected near-gold rows" + for row in rows: + prov = row.provenance + assert prov.parent_checkpoint == "ckpt-abc" + assert prov.source_commit + assert prov.prompt_hash + assert prov.suite_hash + assert prov.gold_visibility_policy == GOLD_VISIBILITY_ACQUISITION_AND_LABELS + assert prov.rollout_config["acquisition"] == "sample_mutation" + # Labels present: SLM-308 distance + SLM-305 legal supervised edit. + assert row.distance_label["kind"] in { + DistanceKind.EXACT.value, + DistanceKind.BOUNDED.value, + DistanceKind.UNKNOWN.value, + } + assert row.supervised_edit is not None + assert "action_name" in row.supervised_edit + + +def test_on_policy_provenance_is_gold_blind_rollout(space, records) -> None: + from slm_training.harnesses.experiments.slm312_state_sources import ( + acquire_on_policy, + ) + + torch_rows = acquire_gold_only(records[:1], space) + model = TreeEditDiffusionModel.from_records( + records[:1], + config=TreeEditDiffusionConfig(**_SMALL_CONFIG), + device="cpu", + ) + rows = acquire_on_policy( + model, + records[:1], + space, + parent_checkpoint="bootstrap-test", + max_states_per_record=3, + ) + assert rows, "expected on-policy rows from beam telemetry" + kinds = {r.kind for r in rows} + assert kinds <= {"visited", "final", "verifier_failure_cone"} + assert "final" in kinds + for row in rows: + assert row.provenance.parent_checkpoint == "bootstrap-test" + assert row.provenance.gold_visibility_policy == GOLD_VISIBILITY_LABELS_ONLY + assert row.provenance.rollout_config["beam_width"] == model.config.beam_width + assert torch_rows # sanity: scaffold reused + + +# --- leak guard: fail closed ------------------------------------------------- + + +def test_leak_guard_fails_closed_on_overlap(space, records) -> None: + rows = acquire_gold_only(records, space) + victim = rows[0] + with pytest.raises(LeakageError): + assert_no_leakage(rows, {victim.fingerprint()}) + # A disjoint held-out set passes. + assert_no_leakage(rows, {"0" * 64}) + # drop_leaked_rows removes the colliding row and the kept set verifies. + kept, dropped = drop_leaked_rows(rows, {victim.fingerprint()}) + assert len(kept) == len(rows) - 1 + assert dropped and dropped[0]["record_id"] == victim.record_id + assert_no_leakage(kept, {victim.fingerprint()}) + + +# --- snapshot immutability / content addressing ------------------------------- + + +def test_snapshot_roundtrip_and_tamper_detection(space, records, tmp_path) -> None: + rows = acquire_gold_only(records, space) + snap = StateSnapshotV1("snap-test", {"k": 1}, rows) + path = snap.write(tmp_path / "rows.jsonl") + back = read_snapshot(path) + assert back.rows == rows + assert back.manifest()["manifest_sha256"] == snap.manifest()["manifest_sha256"] + + # Tamper with one row payload (keep the stored hash): detect on load. + lines = path.read_text(encoding="utf-8").splitlines() + first = json.loads(lines[0]) + first["kind"] = "tampered" + lines[0] = json.dumps(first, sort_keys=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + with pytest.raises(SnapshotIntegrityError): + read_snapshot(path) + + # Tamper with the manifest only: also detected. + path2 = snap.write(tmp_path / "rows2.jsonl") + manifest_path = path2.with_suffix(path2.suffix + ".manifest.json") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["row_sha256"][0] = "0" * 64 + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(SnapshotIntegrityError): + read_snapshot(path2) + + +# --- seedward source: valid intermediates reaching toward the target --------- + + +def test_seedward_produces_valid_intermediates_toward_target(space, records) -> None: + rows = acquire_seedward(records[:1], space) + assert rows, "expected seedward intermediates" + gold = parse_statements(GOLD) + assert gold is not None + seed_fp = source_fingerprint(SEED) + distances: list[float] = [] + for row in rows: + assert row.source == SOURCE_SEEDWARD + statements = parse_statements(row.statements_source) + assert statements is not None + # Every intermediate is a valid, non-seed state strictly closer than + # the seed side start, oracle-verified. + assert row.fingerprint() != seed_fp + label = distance_to_target( + statements, gold, space=space, inventory=list(row.inventory) + ) + assert label.kind is DistanceKind.EXACT + eff = effective_distance(label) + assert eff is not None + distances.append(eff) + # Supervised edit is SLM-305 legal on this state (fail-closed apply) + # or the state is already at the target (STOP supervision). + if row.supervised_edit is not None: + child = space.apply( + statements, edit_from_dict(row.supervised_edit), list(row.inventory) + ) + assert child is not None + assert row.child_source == render_statements(child) + # The walk is strictly distance-decreasing toward the gold target. + assert distances == sorted(distances, reverse=True) + assert distances[0] > distances[-1] + + +def test_oracle_best_child_strictly_improves(space) -> None: + seed = parse_statements(SEED) + gold = parse_statements(GOLD) + assert seed is not None and gold is not None + inventory = [":hero.title", ":cta.label"] + found = oracle_best_child(seed, gold, space, inventory) + assert found is not None + edit, child, label = found + d_seed = effective_distance( + distance_to_target(seed, gold, space=space, inventory=inventory) + ) + d_child = effective_distance( + distance_to_target(child, gold, space=space, inventory=inventory) + ) + # The child is always oracle-measurable; when the parent is measurable + # too, the child is strictly closer (the seed is UNKNOWN at the shallow + # training budgets, so the comparison is conditional). + assert d_child is not None + if d_seed is not None: + assert d_child < d_seed + assert label.kind is DistanceKind.EXACT + # The returned edit is legal and reproduces the child exactly. + assert space.apply(seed, edit, inventory) is not None + assert render_statements(space.apply(seed, edit, inventory)) == render_statements( + child + ) + + +# --- mixture arms: shared budgets, isolation ---------------------------------- + + +def test_mixture_arms_share_budgets_and_sources(space, records) -> None: + rows = acquire_gold_only(records, space) + acquire_seedward(records, space) + budget = 6 + for arm, weights in PREREGISTERED_ARMS.items(): + selected = build_arm_rows(rows, arm, per_arm_budget=budget, seed=0) + assert len(selected) <= budget + sources = {r.source for r in selected} + expected = {s for s, w in weights.items() if w > 0} + assert sources <= expected + if arm != "mixed": + assert sources <= {arm} + # Determinism: same seed, same rows. + again = build_arm_rows(rows, "mixed", per_arm_budget=budget, seed=0) + first = build_arm_rows(rows, "mixed", per_arm_budget=budget, seed=0) + assert [r.sha256() for r in again] == [r.sha256() for r in first] + # Unknown arm is rejected (predeclared set only). + with pytest.raises(ValueError, match="unknown arm"): + build_arm_rows(rows, "sneaky_extra_arm", per_arm_budget=budget) + + +# --- determinism of acquisition ----------------------------------------------- + + +def test_acquisition_deterministic(space, records) -> None: + a = acquire_gold_only(records, space) + acquire_seedward(records, space) + clear_caches() + b = acquire_gold_only(records, space) + acquire_seedward(records, space) + assert [r.sha256() for r in a] == [r.sha256() for r in b] + + +# --- loss consumes rows (wiring) ---------------------------------------------- + + +def test_state_supervision_loss_runs_and_backprops(space, records) -> None: + rows = acquire_gold_only(records, space) + model = TreeEditDiffusionModel.from_records( + records, + config=TreeEditDiffusionConfig(**_SMALL_CONFIG), + device="cpu", + ) + record_by_id = {r.id: r for r in records} + loss = state_supervision_loss(model, rows[:4], record_by_id) + assert float(loss.detach()) >= 0.0 + loss.backward() + metrics = model.last_training_metrics + assert metrics["rows"] == 4.0 + assert "action" in metrics and "value" in metrics From f6b173105eb2f3e2e5fa2b5fea7e3af452cc2b83 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:51:01 -0500 Subject: [PATCH 08/18] SLM-314: preserve multiple hard-valid AST modes with winner-take-all training - Frozen multi-mode dataset (8 prompts x 2 verifier-accepted modes; canonical alpha-invariant AST fingerprints, sha256 rows + tamper-evident manifest; prompt count declared below ship-gate scale). - WTA/MCL loss: backpropagate only the min-loss mode per example + preregistered floor term (eps=0.1) on losing modes; durable per-example selection telemetry. No set-FTPO machinery. - Two-mode synthetic fixture proves single-gold CE collapses (p(mode B)=0.00004) while WTA retains both modes (p(B)=0.0083, p(A)=0.826 dominant); multi-gold spreads 50% mass to invalid hybrids. - Matched arms {single_gold, multi_gold, wta} identical budgets: coverage 0.750/0.500/0.500, hard-valid decode 1.0, v2 regression 0.000. Verdict rejected per preregistered coverage gate (WTA gain -0.25); at this scale single-gold lifts the sibling mode too. - 16 new tests; slm314 component v1. --- .../iter-slm314-winner-take-all-20260724.json | 844 ++++++++++++++++++ .../iter-slm314-winner-take-all-20260724.md | 36 + scripts/run_slm314_winner_take_all.py | 507 +++++++++++ .../experiments/slm314_winner_take_all.py | 547 ++++++++++++ .../data/slm314_multimode/modes.jsonl | 8 + .../modes.jsonl.manifest.json | 19 + src/slm_training/resources/versions.json | 20 + .../test_slm314_winner_take_all.py | 244 +++++ 8 files changed, 2225 insertions(+) create mode 100644 docs/design/iter-slm314-winner-take-all-20260724.json create mode 100644 docs/design/iter-slm314-winner-take-all-20260724.md create mode 100644 scripts/run_slm314_winner_take_all.py create mode 100644 src/slm_training/harnesses/experiments/slm314_winner_take_all.py create mode 100644 src/slm_training/resources/data/slm314_multimode/modes.jsonl create mode 100644 src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json create mode 100644 tests/test_harnesses/experiments/test_slm314_winner_take_all.py diff --git a/docs/design/iter-slm314-winner-take-all-20260724.json b/docs/design/iter-slm314-winner-take-all-20260724.json new file mode 100644 index 000000000..cee2da10c --- /dev/null +++ b/docs/design/iter-slm314-winner-take-all-20260724.json @@ -0,0 +1,844 @@ +{ + "experiment": "slm314-winner-take-all", + "issue": "SLM-314", + "question": "Does winner-take-all (MCL) training with a preregistered floor term retain more hard-valid canonical AST modes than single-gold CE without a semantic regression, at equal budget?", + "preregistered_thresholds": { + "floor_epsilon": 0.1, + "retained_improvement_min": 0.0, + "retention_gap_margin": 0.5, + "semantic_regression_max_points": 0.05, + "verdict_rule": "wta_preserves_modes iff coverage(wta) > coverage(single_gold) AND v2 semantic verdict regression vs single_gold <= 0.05, else rejected" + }, + "arms": [ + "single_gold", + "multi_gold", + "wta" + ], + "config": { + "steps": 5, + "lr": 0.003, + "seed": 0, + "n_prompts": 8, + "n_modes": 16, + "dataset": "src/slm_training/resources/data/slm314_multimode/modes.jsonl", + "dataset_manifest": { + "dataset_schema": "slm314-multimode-dataset/v1", + "n_prompts": 8, + "n_modes": 16, + "modes_per_prompt": [ + 2 + ], + "row_sha256": [ + "17bd182f4bddf979caabdf1248fe325418a4047575600ae6d10cad1c47314774", + "f6ac316055c35daefbdb864879ff504f269326d8d0bde64fc585b14d8be28882", + "015568fa00c3a12f895aeb58af0b85cb799e28f2c315d42fe02bfb573dbca89e", + "b22959ee91c136cbb9814a20ef9b4cf579924bab79c7c2f15fc3497e6ac9df62", + "35bd9d295405696d90b4748c671460a2db892552d824760b1a170ee56b436347", + "aad565646ad04affe14603d1eb0de57fed8d189e1ed2c7c360bf72ed8c78a5f6", + "708f07f6793b1832d3bee49b1f62fa2ab7ef3157b8de5f938c631888fa056b99", + "4b29cac7b71e9aff55a1914f674bc5f54656e01210bdcb6db714f34abccd73fd" + ], + "manifest_sha256": "10d1edc939cc74c159858bf423aa3b457d691526efc41a125c50c5e97c7d42f9" + }, + "fixture_config_overrides": { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 3, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3 + }, + "torch_num_threads": 1 + }, + "synthetic_two_mode": { + "probabilities": { + "single_gold": { + "p_mode_a": 0.9873679876327515, + "p_mode_b": 4.0145620005205274e-05, + "p_invalid_hybrid": 0.012591822072863579 + }, + "multi_gold": { + "p_mode_a": 0.25, + "p_mode_b": 0.25, + "p_invalid_hybrid": 0.5 + }, + "wta": { + "p_mode_a": 0.8264458775520325, + "p_mode_b": 0.008264509029686451, + "p_invalid_hybrid": 0.16528968513011932 + } + }, + "assertions": { + "single_gold_collapses": true, + "wta_retains_loser_mode": true, + "wta_winner_dominant": true, + "multi_gold_hybrid_invalid": true + }, + "all_assertions_pass": true + }, + "arm_results": { + "single_gold": { + "per_mode": { + "n_modes": 16, + "n_retained": 12, + "coverage": 0.75, + "collapse_indicator": 0.25, + "per_group": [ + { + "group_id": "mm_00", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "80c5eda1a6896342816a9794926f6f9595c3c2b2f4a36b5beb68cb7fd91b3341", + "eval_loss": 8.701364517211914, + "baseline_loss": 8.232380867004395, + "improvement": -0.46898365020751953, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "6c454c79cac92994ad15f07e1e43019515d4df7a4dab2b09ab88a7133a0ba913", + "eval_loss": 7.9676361083984375, + "baseline_loss": 9.193222045898438, + "improvement": 1.2255859375, + "retained": true + } + ] + }, + { + "group_id": "mm_01", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "8c94a6a1e407ac005e58301670afa9aaccd0c2b1a04552693289f51a5a48ddf3", + "eval_loss": 4.781002521514893, + "baseline_loss": 5.63767147064209, + "improvement": 0.8566689491271973, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "8b2366208036fe324c2584d98e62cd697d1e0c2e1d2fc4461f19788d6510d691", + "eval_loss": 1.9130592346191406, + "baseline_loss": 2.660956621170044, + "improvement": 0.7478973865509033, + "retained": true + } + ] + }, + { + "group_id": "mm_02", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "4d0cafaed2d1d426605275844c7ffe9cbccb8c017cf18b7c4bff0932199f6fc8", + "eval_loss": 4.843299865722656, + "baseline_loss": 5.75419807434082, + "improvement": 0.9108982086181641, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "29b4a54a4aa6b58ea3cd3ef25a8b282da7a08459de3959cd97443f1eec8e5d2d", + "eval_loss": 6.901802062988281, + "baseline_loss": 8.262958526611328, + "improvement": 1.3611564636230469, + "retained": true + } + ] + }, + { + "group_id": "mm_03", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "5a798e31a8ce98a2a1a616a7e61531c632036bfc150d151bd775edc8903bff5b", + "eval_loss": 11.471742630004883, + "baseline_loss": 12.366077423095703, + "improvement": 0.8943347930908203, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "97b12398d55fd0836abcb35011122985efeb8dfbaf47d9d8d211f331aa598952", + "eval_loss": 4.823297500610352, + "baseline_loss": 6.045340538024902, + "improvement": 1.2220430374145508, + "retained": true + } + ] + }, + { + "group_id": "mm_04", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "b8528408c1ecc2292846532b492e323dec85034f4d98c445a1bfcb22d5120f58", + "eval_loss": 4.774424076080322, + "baseline_loss": 5.668830394744873, + "improvement": 0.8944063186645508, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "9a2d5f9922a1612e18ac88aaed8a1c807f9d61f214ffab1787ede721e85db843", + "eval_loss": 5.237826347351074, + "baseline_loss": 6.058230876922607, + "improvement": 0.8204045295715332, + "retained": true + } + ] + }, + { + "group_id": "mm_05", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "13b7964bf2252bdbe82c5ce990ada9242baffb0f3b0b269be614ecefeedf6dd9", + "eval_loss": 6.782610893249512, + "baseline_loss": 12.469882011413574, + "improvement": 5.6872711181640625, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "1b9cd54f0cf1b9a5bac1f2c7bc8120ab78bccc0f7dc7c247705e065e7591c89a", + "eval_loss": 1.9315738677978516, + "baseline_loss": 2.6750247478485107, + "improvement": 0.7434508800506592, + "retained": false + } + ] + }, + { + "group_id": "mm_06", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "de0096e262a8a20e11bb2338b2b99f3b38fca2034d657bf90c39ee7287d7d958", + "eval_loss": 7.448744297027588, + "baseline_loss": 12.19206714630127, + "improvement": 4.743322849273682, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "54f7897efa57c43a2166b33229c3b1a8f043c020d84d78ef193c2aa4bfc0fd88", + "eval_loss": 4.377158164978027, + "baseline_loss": 6.348398208618164, + "improvement": 1.9712400436401367, + "retained": false + } + ] + }, + { + "group_id": "mm_07", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "d60cd88bf91091b9ffc0087cf970ba229baa342ba8b955b32c4a906da8e881a3", + "eval_loss": 4.873033046722412, + "baseline_loss": 6.304152488708496, + "improvement": 1.431119441986084, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "c7f2a0d70546000767905aa498e7eaea65cdbfe66095ad6e9d93eab629e01969", + "eval_loss": 8.81863784790039, + "baseline_loss": 12.366247177124023, + "improvement": 3.547609329223633, + "retained": true + } + ] + } + ] + }, + "decode": { + "n_prompts": 8, + "hard_valid_rate": 1.0, + "mode_hit_rate": 0.125, + "v2_verdict_rate": 0.0, + "per_group": [ + { + "group_id": "mm_00", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_01", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_02", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_03", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_04", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_05", + "valid": true, + "mode_hit": true, + "v2_verdict": false + }, + { + "group_id": "mm_06", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_07", + "valid": true, + "mode_hit": false, + "v2_verdict": false + } + ] + }, + "coverage": 0.75, + "collapse_indicator": 0.25, + "training_metrics": { + "action": 2.8104822635650635, + "stmt": 2.094724178314209, + "comp": 3.1520068645477295, + "slot": 0.7475554347038269, + "value": 0.013868428766727448, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + }, + "multi_gold": { + "per_mode": { + "n_modes": 16, + "n_retained": 8, + "coverage": 0.5, + "collapse_indicator": 0.5, + "per_group": [ + { + "group_id": "mm_00", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "80c5eda1a6896342816a9794926f6f9595c3c2b2f4a36b5beb68cb7fd91b3341", + "eval_loss": 7.716122627258301, + "baseline_loss": 8.232380867004395, + "improvement": 0.5162582397460938, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "6c454c79cac92994ad15f07e1e43019515d4df7a4dab2b09ab88a7133a0ba913", + "eval_loss": 6.424439907073975, + "baseline_loss": 9.193222045898438, + "improvement": 2.768782138824463, + "retained": true + } + ] + }, + { + "group_id": "mm_01", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "8c94a6a1e407ac005e58301670afa9aaccd0c2b1a04552693289f51a5a48ddf3", + "eval_loss": 3.271097183227539, + "baseline_loss": 5.63767147064209, + "improvement": 2.366574287414551, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "8b2366208036fe324c2584d98e62cd697d1e0c2e1d2fc4461f19788d6510d691", + "eval_loss": 2.3212692737579346, + "baseline_loss": 2.660956621170044, + "improvement": 0.3396873474121094, + "retained": false + } + ] + }, + { + "group_id": "mm_02", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "4d0cafaed2d1d426605275844c7ffe9cbccb8c017cf18b7c4bff0932199f6fc8", + "eval_loss": 3.285755157470703, + "baseline_loss": 5.75419807434082, + "improvement": 2.468442916870117, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "29b4a54a4aa6b58ea3cd3ef25a8b282da7a08459de3959cd97443f1eec8e5d2d", + "eval_loss": 7.695723056793213, + "baseline_loss": 8.262958526611328, + "improvement": 0.5672354698181152, + "retained": false + } + ] + }, + { + "group_id": "mm_03", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "5a798e31a8ce98a2a1a616a7e61531c632036bfc150d151bd775edc8903bff5b", + "eval_loss": 7.438523292541504, + "baseline_loss": 12.366077423095703, + "improvement": 4.927554130554199, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "97b12398d55fd0836abcb35011122985efeb8dfbaf47d9d8d211f331aa598952", + "eval_loss": 5.40690803527832, + "baseline_loss": 6.045340538024902, + "improvement": 0.638432502746582, + "retained": false + } + ] + }, + { + "group_id": "mm_04", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "b8528408c1ecc2292846532b492e323dec85034f4d98c445a1bfcb22d5120f58", + "eval_loss": 3.2084126472473145, + "baseline_loss": 5.668830394744873, + "improvement": 2.4604177474975586, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "9a2d5f9922a1612e18ac88aaed8a1c807f9d61f214ffab1787ede721e85db843", + "eval_loss": 7.189324855804443, + "baseline_loss": 6.058230876922607, + "improvement": -1.131093978881836, + "retained": false + } + ] + }, + { + "group_id": "mm_05", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "13b7964bf2252bdbe82c5ce990ada9242baffb0f3b0b269be614ecefeedf6dd9", + "eval_loss": 9.23392105102539, + "baseline_loss": 12.469882011413574, + "improvement": 3.2359609603881836, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "1b9cd54f0cf1b9a5bac1f2c7bc8120ab78bccc0f7dc7c247705e065e7591c89a", + "eval_loss": 2.3135123252868652, + "baseline_loss": 2.6750247478485107, + "improvement": 0.3615124225616455, + "retained": false + } + ] + }, + { + "group_id": "mm_06", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "de0096e262a8a20e11bb2338b2b99f3b38fca2034d657bf90c39ee7287d7d958", + "eval_loss": 8.623126029968262, + "baseline_loss": 12.19206714630127, + "improvement": 3.568941116333008, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "54f7897efa57c43a2166b33229c3b1a8f043c020d84d78ef193c2aa4bfc0fd88", + "eval_loss": 5.369912624359131, + "baseline_loss": 6.348398208618164, + "improvement": 0.9784855842590332, + "retained": false + } + ] + }, + { + "group_id": "mm_07", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "d60cd88bf91091b9ffc0087cf970ba229baa342ba8b955b32c4a906da8e881a3", + "eval_loss": 5.448168754577637, + "baseline_loss": 6.304152488708496, + "improvement": 0.8559837341308594, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "c7f2a0d70546000767905aa498e7eaea65cdbfe66095ad6e9d93eab629e01969", + "eval_loss": 6.5001139640808105, + "baseline_loss": 12.366247177124023, + "improvement": 5.866133213043213, + "retained": true + } + ] + } + ] + }, + "decode": { + "n_prompts": 8, + "hard_valid_rate": 1.0, + "mode_hit_rate": 0.0, + "v2_verdict_rate": 0.0, + "per_group": [ + { + "group_id": "mm_00", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_01", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_02", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_03", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_04", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_05", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_06", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_07", + "valid": true, + "mode_hit": false, + "v2_verdict": false + } + ] + }, + "coverage": 0.5, + "collapse_indicator": 0.5, + "training_metrics": { + "action": 2.5397448539733887, + "stmt": 1.467846393585205, + "comp": 1.6208996772766113, + "slot": 0.8510135412216187, + "value": 0.02060924842953682, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + }, + "wta": { + "per_mode": { + "n_modes": 16, + "n_retained": 8, + "coverage": 0.5, + "collapse_indicator": 0.5, + "per_group": [ + { + "group_id": "mm_00", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "80c5eda1a6896342816a9794926f6f9595c3c2b2f4a36b5beb68cb7fd91b3341", + "eval_loss": 7.4052324295043945, + "baseline_loss": 8.232380867004395, + "improvement": 0.8271484375, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "6c454c79cac92994ad15f07e1e43019515d4df7a4dab2b09ab88a7133a0ba913", + "eval_loss": 7.391336441040039, + "baseline_loss": 9.193222045898438, + "improvement": 1.8018856048583984, + "retained": true + } + ] + }, + { + "group_id": "mm_01", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "8c94a6a1e407ac005e58301670afa9aaccd0c2b1a04552693289f51a5a48ddf3", + "eval_loss": 2.9950571060180664, + "baseline_loss": 5.63767147064209, + "improvement": 2.6426143646240234, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "8b2366208036fe324c2584d98e62cd697d1e0c2e1d2fc4461f19788d6510d691", + "eval_loss": 2.2241077423095703, + "baseline_loss": 2.660956621170044, + "improvement": 0.43684887886047363, + "retained": false + } + ] + }, + { + "group_id": "mm_02", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "4d0cafaed2d1d426605275844c7ffe9cbccb8c017cf18b7c4bff0932199f6fc8", + "eval_loss": 3.1677842140197754, + "baseline_loss": 5.75419807434082, + "improvement": 2.586413860321045, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "29b4a54a4aa6b58ea3cd3ef25a8b282da7a08459de3959cd97443f1eec8e5d2d", + "eval_loss": 7.574008941650391, + "baseline_loss": 8.262958526611328, + "improvement": 0.6889495849609375, + "retained": false + } + ] + }, + { + "group_id": "mm_03", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "5a798e31a8ce98a2a1a616a7e61531c632036bfc150d151bd775edc8903bff5b", + "eval_loss": 8.58497428894043, + "baseline_loss": 12.366077423095703, + "improvement": 3.7811031341552734, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "97b12398d55fd0836abcb35011122985efeb8dfbaf47d9d8d211f331aa598952", + "eval_loss": 4.4556169509887695, + "baseline_loss": 6.045340538024902, + "improvement": 1.5897235870361328, + "retained": false + } + ] + }, + { + "group_id": "mm_04", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "b8528408c1ecc2292846532b492e323dec85034f4d98c445a1bfcb22d5120f58", + "eval_loss": 2.9114532470703125, + "baseline_loss": 5.668830394744873, + "improvement": 2.7573771476745605, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "9a2d5f9922a1612e18ac88aaed8a1c807f9d61f214ffab1787ede721e85db843", + "eval_loss": 6.242021560668945, + "baseline_loss": 6.058230876922607, + "improvement": -0.1837906837463379, + "retained": false + } + ] + }, + { + "group_id": "mm_05", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "13b7964bf2252bdbe82c5ce990ada9242baffb0f3b0b269be614ecefeedf6dd9", + "eval_loss": 9.684586524963379, + "baseline_loss": 12.469882011413574, + "improvement": 2.7852954864501953, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "1b9cd54f0cf1b9a5bac1f2c7bc8120ab78bccc0f7dc7c247705e065e7591c89a", + "eval_loss": 2.2146434783935547, + "baseline_loss": 2.6750247478485107, + "improvement": 0.46038126945495605, + "retained": false + } + ] + }, + { + "group_id": "mm_06", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "de0096e262a8a20e11bb2338b2b99f3b38fca2034d657bf90c39ee7287d7d958", + "eval_loss": 11.33463191986084, + "baseline_loss": 12.19206714630127, + "improvement": 0.8574352264404297, + "retained": true + }, + { + "mode_index": 1, + "canonical_fingerprint": "54f7897efa57c43a2166b33229c3b1a8f043c020d84d78ef193c2aa4bfc0fd88", + "eval_loss": 6.33472204208374, + "baseline_loss": 6.348398208618164, + "improvement": 0.013676166534423828, + "retained": false + } + ] + }, + { + "group_id": "mm_07", + "modes": [ + { + "mode_index": 0, + "canonical_fingerprint": "d60cd88bf91091b9ffc0087cf970ba229baa342ba8b955b32c4a906da8e881a3", + "eval_loss": 4.460836410522461, + "baseline_loss": 6.304152488708496, + "improvement": 1.8433160781860352, + "retained": false + }, + { + "mode_index": 1, + "canonical_fingerprint": "c7f2a0d70546000767905aa498e7eaea65cdbfe66095ad6e9d93eab629e01969", + "eval_loss": 8.877434730529785, + "baseline_loss": 12.366247177124023, + "improvement": 3.4888124465942383, + "retained": true + } + ] + } + ] + }, + "decode": { + "n_prompts": 8, + "hard_valid_rate": 1.0, + "mode_hit_rate": 0.125, + "v2_verdict_rate": 0.0, + "per_group": [ + { + "group_id": "mm_00", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_01", + "valid": true, + "mode_hit": true, + "v2_verdict": false + }, + { + "group_id": "mm_02", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_03", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_04", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_05", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_06", + "valid": true, + "mode_hit": false, + "v2_verdict": false + }, + { + "group_id": "mm_07", + "valid": true, + "mode_hit": false, + "v2_verdict": false + } + ] + }, + "coverage": 0.5, + "collapse_indicator": 0.5, + "training_metrics": { + "action": 3.007995367050171, + "stmt": 1.427188754081726, + "comp": 2.3815364837646484, + "slot": 2.056959390640259, + "value": 0.003755042562261224, + "value_bounded": 0.0, + "value_unknown_excluded": 0.0, + "skipped": 0.0 + } + } + }, + "comparison": { + "coverage_gain_wta_vs_single_gold": -0.25, + "semantic_regression_wta_vs_single_gold": 0.0, + "coverage_multi_gold": 0.5 + }, + "threshold_checks": { + "coverage_ok": false, + "semantic_ok": true, + "synthetic_ok": true + }, + "telemetry_path": "outputs/slm314/wta_telemetry.jsonl", + "verdict": "rejected", + "wall_seconds": 75.39187614799448, + "honesty": "Fixture-scale: 8 prompts x 2 modes is far below any ship-gate prompt count \u2014 mechanism evidence only, not a production ship claim. Mode identity is alpha-invariant canonical AST fingerprints (never serialization strings); the frozen dataset is hash-verified before training. All arms share model init, steps, optimizer, per-step prompt coverage, and seeds. Decode is deterministic beam search (unique valid ASTs per prompt == 1 by construction); coverage is measured by deterministic per-mode eval loss instead. The SLM-130 ambiguity sets are wiring reports without a committed multi-mode corpus, so this dataset is synthesized and frozen in-repo.", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "0dd815193a4fccb41f229751393ed18a989718d7", + "code_dirty": true, + "components": { + "harness.experiments.slm314_winner_take_all": "v1", + "harness.experiments.slm299_edit_reachability": "v4" + }, + "stamped_at": "2026-07-24T23:47:05.065449+00:00" + } +} diff --git a/docs/design/iter-slm314-winner-take-all-20260724.md b/docs/design/iter-slm314-winner-take-all-20260724.md new file mode 100644 index 000000000..3776ed4d4 --- /dev/null +++ b/docs/design/iter-slm314-winner-take-all-20260724.md @@ -0,0 +1,36 @@ +# SLM-314 (LAR2-05): winner-take-all over multi-mode gold + +**Verdict: `rejected`** (fixture-scale matched arms; not a ship claim) + +## Preregistered (locked before results) + +- arms: `["single_gold", "multi_gold", "wta"]` — identical model init, steps, optimizer, per-step prompt coverage, seeds +- floor epsilon: 0.1; retained iff eval-loss improvement > 0.0 vs matched baseline and within 0.5 of the prompt's best-improved mode +- rule: wta_preserves_modes iff coverage(wta) > coverage(single_gold) AND v2 semantic verdict regression vs single_gold <= 0.05, else rejected +- frozen dataset: `src/slm_training/resources/data/slm314_multimode/modes.jsonl` (8 prompts x 2 modes, manifest `10d1edc939cc74c1…`) + +## Synthetic two-mode proof (deterministic) + +| arm | p(mode A) | p(mode B) | p(invalid hybrid) | +| --- | --- | --- | --- | +| single_gold | 0.987 | 0.000 | 0.013 | +| multi_gold | 0.250 | 0.250 | 0.500 | +| wta | 0.826 | 0.008 | 0.165 | + +Assertions: single_gold_collapses=PASS, wta_retains_loser_mode=PASS, wta_winner_dominant=PASS, multi_gold_hybrid_invalid=PASS + +## Matched arms (real tree-edit model, frozen multi-mode dataset) + +| arm | mode coverage | collapse | hard-valid | mode-hit | v2 verdict | +| --- | --- | --- | --- | --- | --- | +| single_gold | 0.750 | 0.250 | 1.000 | 0.125 | 0.000 | +| multi_gold | 0.500 | 0.500 | 1.000 | 0.000 | 0.000 | +| wta | 0.500 | 0.500 | 1.000 | 0.125 | 0.000 | + +Coverage gain (wta − single_gold): **-0.250**; v2 semantic regression: **0.000** (budget 0.05). Multi-gold coverage: 0.500. + +WTA per-step selected-mode telemetry: `outputs/slm314/wta_telemetry.jsonl`. + +## Honesty + +Fixture-scale: 8 prompts x 2 modes is far below any ship-gate prompt count — mechanism evidence only, not a production ship claim. Mode identity is alpha-invariant canonical AST fingerprints (never serialization strings); the frozen dataset is hash-verified before training. All arms share model init, steps, optimizer, per-step prompt coverage, and seeds. Decode is deterministic beam search (unique valid ASTs per prompt == 1 by construction); coverage is measured by deterministic per-mode eval loss instead. The SLM-130 ambiguity sets are wiring reports without a committed multi-mode corpus, so this dataset is synthesized and frozen in-repo. diff --git a/scripts/run_slm314_winner_take_all.py b/scripts/run_slm314_winner_take_all.py new file mode 100644 index 000000000..304fcd933 --- /dev/null +++ b/scripts/run_slm314_winner_take_all.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""SLM-314 (LAR2-05): winner-take-all vs single-gold vs multi-gold CE. + +Question: when a prompt has multiple hard-valid canonical AST modes, does +winner-take-all (MCL) training — min-loss mode per example plus a small +preregistered coverage/floor term — retain MORE verifier-accepted modes than +single-gold CE (which drops all but one mode at the dataset level) without a +semantic regression, at equal model / steps / optimizer / seed budget? The +duplicated-example multi-gold CE arm is the averaging baseline. + +Preregistered (locked before any run; written into the payload): + +- frozen dataset: ``src/slm_training/resources/data/slm314_multimode/modes.jsonl`` + (8 prompts x 2 verifier-accepted canonical AST modes; mode identity = + alpha-invariant canonical AST fingerprints + sha256 freeze manifest); +- arms {single_gold, multi_gold, wta} with identical model init, steps, + optimizer, per-step prompt coverage, and seeds; +- WTA floor epsilon 0.1; a frozen mode counts as RETAINED when its + deterministic eval loss IMPROVES vs the matched untrained baseline (same + init seed, same fixed eval rng) by more than 0.0 AND keeps pace with its + prompt's best-improved mode within 0.5 loss units; +- primary verdict ``wta_preserves_modes`` iff coverage(wta) > + coverage(single_gold) AND v2 semantic verdict regression vs single_gold + <= 5 points; else ``rejected`` — honestly computed, never narrated; +- synthetic two-mode fixture assertions (collapse/retain/hybrid thresholds in + ``slm314_winner_take_all``) must hold as the mechanism proof. + +Evaluation per arm: per-mode retention (deterministic eval loss), mode +coverage, collapse indicator, decode hard-valid rate, decoded-AST mode-hit +rate (canonical fingerprint match against the prompt's frozen modes), and v2 +semantic verdict rate (``binding_aware_meaningful_v2``). Fixture-scale: 8 +prompts is far below any ship-gate prompt count — this is mechanism evidence, +not a ship claim. + +Writes ``docs/design/iter-slm314-winner-take-all-20260724.{json,md}`` and +durable WTA telemetry under ``outputs/slm314/``. + +Example: + python -m scripts.run_slm314_winner_take_all --steps 10 +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import torch + +from slm_training.dsl.schema import ExampleRecord +from slm_training.evals.meaningful_program import binding_aware_meaningful_v2 +from slm_training.harnesses.experiments.slm314_winner_take_all import ( + DEFAULT_RESOURCE, + EXPERIMENT_ID, + FLOOR_EPSILON, + PromptGroupV1, + _seed_int, + dataset_manifest, + mode_supervision_loss, + read_frozen_dataset, + run_synthetic_two_mode, + single_gold_groups, + synthetic_assertions, + winner_take_all_loss, +) +from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + _is_valid, + parse_statements, +) +from slm_training.harnesses.experiments.slm312_state_sources import ast_fingerprint +from slm_training.versioning import build_version_stamp + +DEFAULT_JSON_OUT = Path("docs/design/iter-slm314-winner-take-all-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm314-winner-take-all-20260724.md") +DEFAULT_TELEMETRY_OUT = Path("outputs/slm314/wta_telemetry.jsonl") + +ARMS = ("single_gold", "multi_gold", "wta") + +# Tiny fixture model/decode config — identical across ALL arms and declared in +# the payload; it only bounds fixture wall-clock (the run cap), never differs +# by arm. +FIXTURE_CONFIG_OVERRIDES = { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 3, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3, +} + +# PREREGISTERED thresholds (locked before any run; deviations are append-only +# and exploratory). +PREREGISTERED_THRESHOLDS = { + "floor_epsilon": FLOOR_EPSILON, + "retained_improvement_min": 0.0, + "retention_gap_margin": 0.5, + "semantic_regression_max_points": 0.05, + "verdict_rule": ( + "wta_preserves_modes iff coverage(wta) > coverage(single_gold) AND " + "v2 semantic verdict regression vs single_gold <= 0.05, else rejected" + ), +} + + +def build_model(records: list[ExampleRecord], *, seed: int) -> TreeEditDiffusionModel: + """Identical model init for every arm (same tokenizer texts, same seed).""" + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + seed=seed, + # mutation_count value labels: the SLM-308 bounded-distance oracle is + # too expensive for this fixture's per-mode WTA loss (measured 5x + # wall-clock); arms stay matched and the mode is declared. + value_label_mode="mutation_count", + context_backend="scratch", + **FIXTURE_CONFIG_OVERRIDES, + ) + return TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + + +def train_arm( + arm: str, + groups: list[PromptGroupV1], + records: list[ExampleRecord], + *, + steps: int, + lr: float, + seed: int, + telemetry_rows: list[dict] | None = None, +) -> TreeEditDiffusionModel: + """One arm. Every step covers every prompt group exactly once for all + arms (identical prompt budget); arms differ ONLY in loss weighting: + single_gold = one mode per prompt, multi_gold = plain CE over duplicated + per-mode examples, wta = min-loss mode + floor term.""" + model = build_model(records, seed=seed) + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=lr) + sg_records = [r for g in single_gold_groups(groups) for r in g.records()] + mg_records = [r for g in groups for r in g.records()] + model.train() + for step in range(steps): + if arm == "single_gold": + loss = model.training_loss(sg_records) + elif arm == "multi_gold": + loss = model.training_loss(mg_records) + else: + total = None + for group in groups: + group_loss, telemetry = winner_take_all_loss( + model, group, floor_epsilon=FLOOR_EPSILON, seed=seed + step + ) + total = group_loss if total is None else total + group_loss + if telemetry_rows is not None: + telemetry_rows.append({"step": step, **telemetry}) + loss = total + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +@torch.no_grad() +def per_mode_eval_losses( + model: TreeEditDiffusionModel, groups: list[PromptGroupV1] +) -> dict[tuple[str, str], float]: + """Deterministic per-mode eval loss (fixed eval rng, shared across arms).""" + losses: dict[tuple[str, str], float] = {} + for group in groups: + for mode, record in zip(group.modes, group.records()): + losses[(group.group_id, mode.canonical_fingerprint)] = float( + mode_supervision_loss( + model, + record, + seed=_seed_int( + "slm314-eval", group.group_id, mode.canonical_fingerprint + ), + ).detach().cpu() + ) + return losses + + +def eval_per_mode( + model: TreeEditDiffusionModel, + groups: list[PromptGroupV1], + baseline: dict[tuple[str, str], float], +) -> dict: + """A mode is RETAINED when its deterministic eval loss improves on the + matched untrained baseline (same init seed, same eval rng) by more than + the preregistered margin AND it keeps pace with its prompt's + best-improved mode within ``retention_gap_margin`` loss units.""" + margin = PREREGISTERED_THRESHOLDS["retained_improvement_min"] + gap = PREREGISTERED_THRESHOLDS["retention_gap_margin"] + losses = per_mode_eval_losses(model, groups) + per_group: list[dict] = [] + retained = 0 + total = 0 + for group in groups: + improvements = [ + baseline[(group.group_id, m.canonical_fingerprint)] + - losses[(group.group_id, m.canonical_fingerprint)] + for m in group.modes + ] + best = max(improvements) + rows = [] + for mode, improvement in zip(group.modes, improvements): + key = (group.group_id, mode.canonical_fingerprint) + loss = losses[key] + kept = improvement > margin and improvement >= best - gap + retained += int(kept) + total += 1 + rows.append( + { + "mode_index": mode.mode_index, + "canonical_fingerprint": mode.canonical_fingerprint, + "eval_loss": loss, + "baseline_loss": baseline[key], + "improvement": improvement, + "retained": kept, + } + ) + per_group.append({"group_id": group.group_id, "modes": rows}) + coverage = retained / max(total, 1) + return { + "n_modes": total, + "n_retained": retained, + "coverage": coverage, + "collapse_indicator": 1.0 - coverage, + "per_group": per_group, + } + + +@torch.no_grad() +def eval_decode( + model: TreeEditDiffusionModel, groups: list[PromptGroupV1] +) -> dict: + """Decode once per prompt (beam decode is deterministic): hard-valid rate, + mode-hit rate (decoded AST fingerprint in the prompt's frozen mode set), + and v2 semantic verdict rate.""" + n_valid = 0 + n_hit = 0 + n_v2 = 0 + rows = [] + for group in groups: + inventory = [p if p.startswith(":") else f":{p}" for p in group.modes[0].placeholders] + prompt = model._format_context(group.prompt, slot_contract=inventory) + ctx, ctx_pad = model._encode_context([prompt]) + text, _evidence = model._decode_one(ctx, ctx_pad, inventory) + valid = bool(text) and _is_valid(text) + hit = False + if valid: + statements = parse_statements(text) + if statements is not None: + fp = ast_fingerprint(statements) + hit = fp in {m.canonical_fingerprint for m in group.modes} + report = binding_aware_meaningful_v2( + text, + record=ExampleRecord( + id=group.group_id, + prompt=group.prompt, + openui=group.modes[0].source, + placeholders=list(inventory), + ), + ) + n_valid += int(valid) + n_hit += int(hit) + n_v2 += int(report.verdict) + rows.append( + { + "group_id": group.group_id, + "valid": valid, + "mode_hit": hit, + "v2_verdict": bool(report.verdict), + } + ) + n = max(len(groups), 1) + return { + "n_prompts": len(groups), + "hard_valid_rate": n_valid / n, + "mode_hit_rate": n_hit / n, + "v2_verdict_rate": n_v2 / n, + "per_group": rows, + } + + +def build_report( + groups: list[PromptGroupV1], + *, + steps: int, + lr: float, + seed: int, + telemetry_out: Path, +) -> dict: + torch.set_num_threads(1) # tiny CPU fixture; identical for every arm + start = time.perf_counter() + records = [r for g in groups for r in g.records()] + + # Matched untrained baseline (identical init seed/config): retention is + # measured as per-mode eval-loss IMPROVEMENT against this baseline. + baseline = per_mode_eval_losses(build_model(records, seed=seed), groups) + + telemetry_rows: list[dict] = [] + arms: dict[str, dict] = {} + for arm in ARMS: + model = train_arm( + arm, + groups, + records, + steps=steps, + lr=lr, + seed=seed, + telemetry_rows=telemetry_rows if arm == "wta" else None, + ) + per_mode = eval_per_mode(model, groups, baseline) + decode = eval_decode(model, groups) + arms[arm] = { + "per_mode": per_mode, + "decode": decode, + "coverage": per_mode["coverage"], + "collapse_indicator": per_mode["collapse_indicator"], + "training_metrics": model.last_training_metrics, + } + + telemetry_out.parent.mkdir(parents=True, exist_ok=True) + with telemetry_out.open("w", encoding="utf-8") as fh: + for row in telemetry_rows: + fh.write(json.dumps(row, sort_keys=True) + "\n") + + synthetic = run_synthetic_two_mode() + synthetic_checks = synthetic_assertions(synthetic) + + coverage_gain = arms["wta"]["coverage"] - arms["single_gold"]["coverage"] + semantic_regression = ( + arms["single_gold"]["decode"]["v2_verdict_rate"] + - arms["wta"]["decode"]["v2_verdict_rate"] + ) + coverage_ok = coverage_gain > 0.0 + semantic_ok = ( + semantic_regression + <= PREREGISTERED_THRESHOLDS["semantic_regression_max_points"] + ) + verdict = "wta_preserves_modes" if (coverage_ok and semantic_ok) else "rejected" + + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-314", + "question": ( + "Does winner-take-all (MCL) training with a preregistered floor " + "term retain more hard-valid canonical AST modes than single-gold " + "CE without a semantic regression, at equal budget?" + ), + "preregistered_thresholds": PREREGISTERED_THRESHOLDS, + "arms": ARMS, + "config": { + "steps": steps, + "lr": lr, + "seed": seed, + "n_prompts": len(groups), + "n_modes": sum(len(g.modes) for g in groups), + "dataset": str(DEFAULT_RESOURCE), + "dataset_manifest": dataset_manifest(groups), + "fixture_config_overrides": FIXTURE_CONFIG_OVERRIDES, + "torch_num_threads": 1, + }, + "synthetic_two_mode": { + "probabilities": synthetic, + "assertions": synthetic_checks, + "all_assertions_pass": all(synthetic_checks.values()), + }, + "arm_results": arms, + "comparison": { + "coverage_gain_wta_vs_single_gold": coverage_gain, + "semantic_regression_wta_vs_single_gold": semantic_regression, + "coverage_multi_gold": arms["multi_gold"]["coverage"], + }, + "threshold_checks": { + "coverage_ok": coverage_ok, + "semantic_ok": semantic_ok, + "synthetic_ok": all(synthetic_checks.values()), + }, + "telemetry_path": str(telemetry_out), + "verdict": verdict, + "wall_seconds": time.perf_counter() - start, + "honesty": ( + "Fixture-scale: 8 prompts x 2 modes is far below any ship-gate " + "prompt count — mechanism evidence only, not a production ship " + "claim. Mode identity is alpha-invariant canonical AST " + "fingerprints (never serialization strings); the frozen dataset " + "is hash-verified before training. All arms share model init, " + "steps, optimizer, per-step prompt coverage, and seeds. Decode is " + "deterministic beam search (unique valid ASTs per prompt == 1 by " + "construction); coverage is measured by deterministic per-mode " + "eval loss instead. The SLM-130 ambiguity sets are wiring reports " + "without a committed multi-mode corpus, so this dataset is " + "synthesized and frozen in-repo." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm314_winner_take_all", + "harness.experiments.slm299_edit_reachability", + ) + return payload + + +def render_markdown(payload: dict) -> str: + def f(x: float) -> str: + return f"{x:.3f}" + + lines = [ + "# SLM-314 (LAR2-05): winner-take-all over multi-mode gold", + "", + f"**Verdict: `{payload['verdict']}`** (fixture-scale matched arms; not a ship claim)", + "", + "## Preregistered (locked before results)", + "", + f"- arms: `{json.dumps(payload['arms'])}` — identical model init, steps, " + "optimizer, per-step prompt coverage, seeds", + f"- floor epsilon: {payload['preregistered_thresholds']['floor_epsilon']}; " + f"retained iff eval-loss improvement > " + f"{payload['preregistered_thresholds']['retained_improvement_min']} vs matched " + f"baseline and within {payload['preregistered_thresholds']['retention_gap_margin']} " + "of the prompt's best-improved mode", + f"- rule: {payload['preregistered_thresholds']['verdict_rule']}", + f"- frozen dataset: `{payload['config']['dataset']}` " + f"({payload['config']['n_prompts']} prompts x " + f"{payload['config']['n_modes'] // payload['config']['n_prompts']} modes, " + f"manifest `{payload['config']['dataset_manifest']['manifest_sha256'][:16]}…`)", + "", + "## Synthetic two-mode proof (deterministic)", + "", + "| arm | p(mode A) | p(mode B) | p(invalid hybrid) |", + "| --- | --- | --- | --- |", + ] + for arm, probs in payload["synthetic_two_mode"]["probabilities"].items(): + lines.append( + f"| {arm} | {f(probs['p_mode_a'])} | {f(probs['p_mode_b'])} | " + f"{f(probs['p_invalid_hybrid'])} |" + ) + checks = payload["synthetic_two_mode"]["assertions"] + lines += [ + "", + "Assertions: " + ", ".join(f"{k}={'PASS' if v else 'FAIL'}" for k, v in checks.items()), + "", + "## Matched arms (real tree-edit model, frozen multi-mode dataset)", + "", + "| arm | mode coverage | collapse | hard-valid | mode-hit | v2 verdict |", + "| --- | --- | --- | --- | --- | --- |", + ] + for arm in payload["arms"]: + r = payload["arm_results"][arm] + lines.append( + f"| {arm} | {f(r['coverage'])} | {f(r['collapse_indicator'])} | " + f"{f(r['decode']['hard_valid_rate'])} | {f(r['decode']['mode_hit_rate'])} | " + f"{f(r['decode']['v2_verdict_rate'])} |" + ) + cmp_ = payload["comparison"] + lines += [ + "", + f"Coverage gain (wta − single_gold): **{f(cmp_['coverage_gain_wta_vs_single_gold'])}**; " + f"v2 semantic regression: **{f(cmp_['semantic_regression_wta_vs_single_gold'])}** " + f"(budget 0.05). Multi-gold coverage: {f(cmp_['coverage_multi_gold'])}.", + "", + f"WTA per-step selected-mode telemetry: `{payload['telemetry_path']}`.", + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=5) + parser.add_argument("--lr", type=float, default=3e-3) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--dataset", type=Path, default=DEFAULT_RESOURCE) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + parser.add_argument("--telemetry-out", type=Path, default=DEFAULT_TELEMETRY_OUT) + args = parser.parse_args(argv) + + groups = read_frozen_dataset(args.dataset) + payload = build_report( + groups, + steps=args.steps, + lr=args.lr, + seed=args.seed, + telemetry_out=args.telemetry_out, + ) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"verdict={payload['verdict']} " + f"coverage_gain={payload['comparison']['coverage_gain_wta_vs_single_gold']:.3f} " + f"semantic_regression={payload['comparison']['semantic_regression_wta_vs_single_gold']:.3f} " + f"-> {args.json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm314_winner_take_all.py b/src/slm_training/harnesses/experiments/slm314_winner_take_all.py new file mode 100644 index 000000000..8c2045c90 --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm314_winner_take_all.py @@ -0,0 +1,547 @@ +"""SLM-314 (LAR2-05): winner-take-all (WTA/MCL) training over multi-mode gold. + +Problem: for many prompts there are several hard-valid canonical AST modes +(distinct verifier-accepted programs that all satisfy the prompt). Standard +single-gold CE picks ONE mode at the dataset level and gives zero gradient +toward the others (dataset-level mode collapse); duplicated-example +multi-gold CE averages the modes and can converge to an interpolated, +invalid hybrid. Winner-take-all / multiple-choice learning backpropagates +only the MINIMUM-loss mode per example, plus a small preregistered +coverage/floor term on the unused modes so they stay constrained instead of +drifting to zero probability. + +This module provides: + +- a deterministic, frozen fixture-scale multi-mode dataset (8 prompts x 2 + verifier-accepted canonical AST modes each), committed as a resource JSONL + with per-mode alpha-invariant canonical AST fingerprints (SLM-299 + ``_canonical_key`` sha256 via SLM-312 ``ast_fingerprint``) plus a sha256 + freeze manifest — mode identity is NEVER the serialization string; +- :func:`mode_supervision_loss` — the existing ``training_loss`` terms + computed deterministically per mode (dedicated seeded rng, model rng + restored afterwards); +- :func:`winner_take_all_loss` — per example, backpropagate + ``L_winner + floor_epsilon * mean(L_losers)`` with durable per-example + selected-mode + per-mode loss telemetry; +- :func:`run_synthetic_two_mode` — a minimal two-mode analytic fixture + proving single-gold CE collapses while WTA retains both modes (and + multi-gold CE produces invalid hybrids), with deterministic assertions + consumed by the tests and the CLI. + +Set-FTPO machinery is NOT reused (previously rejected). Torch is imported +lazily inside the loss/synthetic functions so dataset-freeze tests stay +cheap. +""" + +from __future__ import annotations + +import hashlib +import json +import random +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Sequence + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm312_state_sources import ( + ast_fingerprint, +) +from slm_training.models.tree_edit_diffusion import ( + _is_valid, + parse_statements, +) + +EXPERIMENT_ID = "slm314-winner-take-all" +DATASET_SCHEMA = "slm314-multimode-dataset/v1" +DEFAULT_RESOURCE = Path("src/slm_training/resources/data/slm314_multimode/modes.jsonl") + +# PREREGISTERED WTA floor weight (locked before any run; deviations are +# append-only and exploratory). The losing mode keeps probability mass +# ~ eps/(1+eps) per supervised factor at equilibrium instead of 0. +FLOOR_EPSILON = 0.1 + +# --------------------------------------------------------------------------- # +# Frozen multi-mode dataset +# --------------------------------------------------------------------------- # + +# (prompt, [(mode_source, placeholders), ...]) — every mode must parse through +# the real validator and have a DISTINCT canonical AST fingerprint. +MULTIMODE_PROGRAMS: list[tuple[str, list[tuple[str, list[str]]]]] = [ + ( + "hero with cta", + [ + ( + 'root = Stack([t, b], "column")\n' + 't = TextContent(":hero.title")\n' + 'b = Button(":cta.label")', + [":hero.title", ":cta.label"], + ), + ( + 'root = Stack([c], "column")\n' + 'c = Card([t, b], "column")\n' + 't = TextContent(":hero.title")\n' + 'b = Button(":cta.label")', + [":hero.title", ":cta.label"], + ), + ], + ), + ( + "simple text", + [ + ('root = Stack([t], "column")\nt = TextContent(":body")', [":body"]), + ('root = Stack([c], "column")\nc = Card([t], "column")\nt = TextContent(":body")', [":body"]), + ], + ), + ( + "form with email and submit", + [ + ( + 'root = Form([i, s], "column")\n' + 'i = TextContent(":form.email")\n' + 's = Button(":form.submit")', + [":form.email", ":form.submit"], + ), + ( + 'root = Form([c], "column")\n' + 'c = Card([i, s], "column")\n' + 'i = TextContent(":form.email")\n' + 's = Button(":form.submit")', + [":form.email", ":form.submit"], + ), + ], + ), + ( + "card with image", + [ + ( + 'root = Stack([c], "column")\n' + 'c = Card([im, t], "column")\n' + 'im = Image(":img.src")\n' + 't = TextContent(":img.caption")', + [":img.src", ":img.caption"], + ), + ( + 'root = Card([im, t], "column")\n' + 'im = Image(":img.src")\n' + 't = TextContent(":img.caption")', + [":img.src", ":img.caption"], + ), + ], + ), + ( + "two texts", + [ + ( + 'root = Stack([a, b], "column")\n' + 'a = TextContent(":title")\n' + 'b = TextContent(":subtitle")', + [":title", ":subtitle"], + ), + ( + 'root = Stack([c], "column")\n' + 'c = Card([a, b], "column")\n' + 'a = TextContent(":title")\n' + 'b = TextContent(":subtitle")', + [":title", ":subtitle"], + ), + ], + ), + ( + "button only", + [ + ('root = Stack([b], "column")\nb = Button(":cta")', [":cta"]), + ('root = Card([b], "column")\nb = Button(":cta")', [":cta"]), + ], + ), + ( + "nested card", + [ + ( + 'root = Stack([c], "column")\n' + 'c = Card([t], "column")\n' + 't = TextContent(":card.body")', + [":card.body"], + ), + ( + 'root = Stack([c], "column")\n' + 'c = Card([inner], "column")\n' + 'inner = Card([t], "column")\n' + 't = TextContent(":card.body")', + [":card.body"], + ), + ], + ), + ( + "form single field", + [ + ('root = Form([i], "column")\ni = TextContent(":q")', [":q"]), + ( + 'root = Form([c], "column")\n' + 'c = Card([i], "column")\n' + 'i = TextContent(":q")', + [":q"], + ), + ], + ), +] + + +@dataclass(frozen=True) +class ModeV1: + """One verifier-accepted canonical AST mode of a prompt.""" + + mode_index: int + source: str + canonical_fingerprint: str # alpha-invariant AST fingerprint (sha256) + placeholders: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["placeholders"] = list(self.placeholders) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "ModeV1": + return cls( + mode_index=int(data["mode_index"]), + source=str(data["source"]), + canonical_fingerprint=str(data["canonical_fingerprint"]), + placeholders=tuple(str(x) for x in data.get("placeholders") or ()), + ) + + +@dataclass(frozen=True) +class PromptGroupV1: + """One prompt with >=2 verifier-accepted canonical AST modes.""" + + group_id: str + prompt: str + modes: tuple[ModeV1, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "group_id": self.group_id, + "prompt": self.prompt, + "modes": [m.to_dict() for m in self.modes], + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "PromptGroupV1": + return cls( + group_id=str(data["group_id"]), + prompt=str(data["prompt"]), + modes=tuple(ModeV1.from_dict(m) for m in data.get("modes") or ()), + ) + + def records(self) -> list[ExampleRecord]: + """One ExampleRecord per mode (identical prompt, distinct gold).""" + return [ + ExampleRecord( + id=f"{self.group_id}__m{mode.mode_index}", + prompt=self.prompt, + openui=mode.source, + placeholders=list(mode.placeholders), + split="train", + ) + for mode in self.modes + ] + + +def _fingerprint_source(source: str) -> str: + """Alpha-invariant canonical AST fingerprint; raises when not valid.""" + if not _is_valid(source): + raise ValueError(f"mode is not verifier-accepted: {source!r}") + statements = parse_statements(source) + if statements is None: + raise ValueError(f"mode does not parse structurally: {source!r}") + return ast_fingerprint(statements) + + +def build_multimode_dataset() -> list[PromptGroupV1]: + """Deterministically construct the multi-mode dataset from the frozen spec. + + Fail-closed: every mode must be verifier-accepted through the real parser + and every prompt must have >=2 DISTINCT canonical AST fingerprints. + """ + groups: list[PromptGroupV1] = [] + for index, (prompt, mode_specs) in enumerate(MULTIMODE_PROGRAMS): + modes: list[ModeV1] = [] + seen: set[str] = set() + for mode_index, (source, placeholders) in enumerate(mode_specs): + fp = _fingerprint_source(source) + if fp in seen: + raise ValueError( + f"prompt {prompt!r} mode {mode_index} duplicates a canonical " + "AST fingerprint — modes must be distinct canonical ASTs" + ) + seen.add(fp) + modes.append( + ModeV1( + mode_index=mode_index, + source=source, + canonical_fingerprint=fp, + placeholders=tuple(placeholders), + ) + ) + if len(modes) < 2: + raise ValueError(f"prompt {prompt!r} has fewer than 2 modes") + groups.append( + PromptGroupV1( + group_id=f"mm_{index:02d}", prompt=prompt, modes=tuple(modes) + ) + ) + return groups + + +def _digest(obj: Any) -> str: + canonical = json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def dataset_manifest(groups: Sequence[PromptGroupV1]) -> dict[str, Any]: + row_hashes = [_digest(g.to_dict()) for g in groups] + return { + "dataset_schema": DATASET_SCHEMA, + "n_prompts": len(groups), + "n_modes": sum(len(g.modes) for g in groups), + "modes_per_prompt": sorted({len(g.modes) for g in groups}), + "row_sha256": row_hashes, + "manifest_sha256": _digest({"schema": DATASET_SCHEMA, "row_sha256": row_hashes}), + } + + +class FreezeIntegrityError(RuntimeError): + """A frozen multi-mode dataset row or manifest failed verification.""" + + +def freeze_dataset(groups: Sequence[PromptGroupV1], path: Path) -> dict[str, Any]: + """Write the dataset as JSONL plus a ``.manifest.json`` sidecar.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + for group in groups: + fh.write(json.dumps(group.to_dict(), sort_keys=True) + "\n") + manifest = dataset_manifest(groups) + path.with_suffix(path.suffix + ".manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return manifest + + +def read_frozen_dataset(path: Path) -> list[PromptGroupV1]: + """Load the frozen dataset, verifying row hashes + manifest (fail-closed).""" + manifest_path = path.with_suffix(path.suffix + ".manifest.json") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + groups: list[PromptGroupV1] = [] + with path.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line: + groups.append(PromptGroupV1.from_dict(json.loads(line))) + if [_digest(g.to_dict()) for g in groups] != list(manifest.get("row_sha256") or []): + raise FreezeIntegrityError(f"row hash mismatch against {manifest_path}") + if dataset_manifest(groups)["manifest_sha256"] != manifest.get("manifest_sha256"): + raise FreezeIntegrityError(f"manifest sha256 mismatch for {manifest_path}") + # Re-verify acceptance + fingerprint identity (never trust strings alone). + for group in groups: + for mode in group.modes: + if _fingerprint_source(mode.source) != mode.canonical_fingerprint: + raise FreezeIntegrityError( + f"{group.group_id} mode {mode.mode_index} fingerprint does not " + "match its re-parsed canonical AST" + ) + return groups + + +def single_gold_groups(groups: Sequence[PromptGroupV1]) -> list[PromptGroupV1]: + """Deterministic single-gold view: per prompt, the mode with the + lexicographically smallest canonical AST fingerprint.""" + reduced: list[PromptGroupV1] = [] + for group in groups: + winner = min(group.modes, key=lambda m: m.canonical_fingerprint) + reduced.append( + PromptGroupV1( + group_id=group.group_id, prompt=group.prompt, modes=(winner,) + ) + ) + return reduced + + +# --------------------------------------------------------------------------- # +# Per-mode deterministic supervision + WTA loss +# --------------------------------------------------------------------------- # + + +def _seed_int(*parts: str) -> int: + digest = hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest() + return int(digest[:16], 16) + + +def mode_supervision_loss(model: Any, record: ExampleRecord, *, seed: int) -> Any: + """The existing ``training_loss`` terms for ONE gold mode, computed with a + dedicated seeded rng (the model's own rng state is saved/restored) so the + per-mode loss is deterministic and identical across arms.""" + saved = model._rng + model._rng = random.Random(seed) + import torch + + try: + with torch.random.fork_rng(devices=[]): + torch.manual_seed(seed) + return model.training_loss([record]) + finally: + model._rng = saved + + +def select_winner(losses: Sequence[float]) -> int: + """Index of the minimum-loss mode; ties break to the lowest mode index + (deterministic).""" + if not losses: + raise ValueError("no mode losses") + best = 0 + for i, value in enumerate(losses): + if value < losses[best]: + best = i + return best + + +def winner_take_all_loss( + model: Any, + group: PromptGroupV1, + *, + floor_epsilon: float = FLOOR_EPSILON, + seed: int = 0, + eval_mode: bool = False, +) -> tuple[Any, dict[str, Any]]: + """WTA/MCL loss for one prompt group: backpropagate the minimum-loss mode + plus ``floor_epsilon * mean(losing-mode losses)`` (coverage/floor term). + + Returns ``(loss, telemetry)``; telemetry records the per-mode losses, the + selected winner fingerprint, and the floor weight (durable evidence). + With ``eval_mode=True`` no gradient-capable loss is required — the same + computation runs under ``torch.no_grad()`` by the caller. + """ + records = group.records() + losses: list[Any] = [] + detached: list[float] = [] + for mode, record in zip(group.modes, records): + loss = mode_supervision_loss( + model, + record, + seed=_seed_int( + "slm314", str(seed), group.group_id, mode.canonical_fingerprint + ), + ) + losses.append(loss) + detached.append(float(loss.detach().cpu())) + winner = select_winner(detached) + losers = [x for i, x in enumerate(losses) if i != winner] + total = losses[winner] + if losers and floor_epsilon > 0: + total = total + floor_epsilon * (sum(losers) / len(losers)) + telemetry = { + "group_id": group.group_id, + "floor_epsilon": floor_epsilon, + "winner_index": winner, + "winner_fingerprint": group.modes[winner].canonical_fingerprint, + "per_mode": [ + { + "mode_index": mode.mode_index, + "canonical_fingerprint": mode.canonical_fingerprint, + "loss": detached[i], + "selected": i == winner, + } + for i, mode in enumerate(group.modes) + ], + "total_loss": float(total.detach().cpu()), + } + return total, telemetry + + +# --------------------------------------------------------------------------- # +# Synthetic two-mode collapse-vs-retain fixture (analytic, deterministic) +# --------------------------------------------------------------------------- # + +# Two positions, two tokens each. Mode A = (0, 0) "Stack/column", mode +# B = (1, 1) "Card/row". Hybrids (0, 1) and (1, 0) are INVALID — the +# token-level average of the two modes. +SYNTHETIC_MODE_A = (0, 0) +SYNTHETIC_MODE_B = (1, 1) + +# PREREGISTERED synthetic assertions (locked before any run): +# - single-gold CE collapses: p(mode B) < SYNTHETIC_COLLAPSE_MAX +# - WTA retains both: p(mode B) >= SYNTHETIC_RETAINED_MIN and p(mode A) > p(B) +# - multi-gold CE averages: invalid-hybrid probability >= SYNTHETIC_HYBRID_MIN +SYNTHETIC_COLLAPSE_MAX = 0.002 +SYNTHETIC_RETAINED_MIN = 0.004 +SYNTHETIC_HYBRID_MIN = 0.3 + + +def run_synthetic_two_mode( + *, + steps: int = 400, + lr: float = 0.2, + floor_epsilon: float = FLOOR_EPSILON, +) -> dict[str, dict[str, float]]: + """Train a 2-position x 2-token softmax under the three arms. + + Full-batch, zero-init logits, no sampling — fully deterministic. Returns + per-arm sequence probabilities ``{p_mode_a, p_mode_b, p_invalid_hybrid}``. + """ + import torch + + def mode_loss(logits: Any, mode: tuple[int, int]) -> Any: + import torch.nn.functional as F + + return sum( + F.cross_entropy(logits[pos : pos + 1], torch.tensor([tok])) + for pos, tok in enumerate(mode) + ) + + def probs(logits: Any) -> dict[str, float]: + p = torch.softmax(logits, dim=-1) + pa = float(p[0, 0] * p[1, 0]) + pb = float(p[0, 1] * p[1, 1]) + hybrid = float(p[0, 0] * p[1, 1] + p[0, 1] * p[1, 0]) + return {"p_mode_a": pa, "p_mode_b": pb, "p_invalid_hybrid": hybrid} + + out: dict[str, dict[str, float]] = {} + for arm in ("single_gold", "multi_gold", "wta"): + torch.manual_seed(0) # identical init across arms (zero anyway) + logits = torch.zeros(2, 2, requires_grad=True) + opt = torch.optim.SGD([logits], lr=lr) + for _ in range(steps): + if arm == "single_gold": + loss = mode_loss(logits, SYNTHETIC_MODE_A) + elif arm == "multi_gold": + loss = 0.5 * ( + mode_loss(logits, SYNTHETIC_MODE_A) + + mode_loss(logits, SYNTHETIC_MODE_B) + ) + else: + la = mode_loss(logits, SYNTHETIC_MODE_A) + lb = mode_loss(logits, SYNTHETIC_MODE_B) + winner, loser = (la, lb) if float(la) <= float(lb) else (lb, la) + loss = winner + floor_epsilon * loser + opt.zero_grad() + loss.backward() + opt.step() + out[arm] = probs(logits.detach()) + return out + + +def synthetic_assertions(result: dict[str, dict[str, float]]) -> dict[str, bool]: + """Deterministic collapse-vs-retain assertions on the synthetic fixture.""" + return { + "single_gold_collapses": ( + result["single_gold"]["p_mode_b"] < SYNTHETIC_COLLAPSE_MAX + ), + "wta_retains_loser_mode": ( + result["wta"]["p_mode_b"] >= SYNTHETIC_RETAINED_MIN + ), + "wta_winner_dominant": ( + result["wta"]["p_mode_a"] > result["wta"]["p_mode_b"] + ), + "multi_gold_hybrid_invalid": ( + result["multi_gold"]["p_invalid_hybrid"] >= SYNTHETIC_HYBRID_MIN + ), + } diff --git a/src/slm_training/resources/data/slm314_multimode/modes.jsonl b/src/slm_training/resources/data/slm314_multimode/modes.jsonl new file mode 100644 index 000000000..c153e4ee6 --- /dev/null +++ b/src/slm_training/resources/data/slm314_multimode/modes.jsonl @@ -0,0 +1,8 @@ +{"group_id": "mm_00", "modes": [{"canonical_fingerprint": "80c5eda1a6896342816a9794926f6f9595c3c2b2f4a36b5beb68cb7fd91b3341", "mode_index": 0, "placeholders": [":hero.title", ":cta.label"], "source": "root = Stack([t, b], \"column\")\nt = TextContent(\":hero.title\")\nb = Button(\":cta.label\")"}, {"canonical_fingerprint": "6c454c79cac92994ad15f07e1e43019515d4df7a4dab2b09ab88a7133a0ba913", "mode_index": 1, "placeholders": [":hero.title", ":cta.label"], "source": "root = Stack([c], \"column\")\nc = Card([t, b], \"column\")\nt = TextContent(\":hero.title\")\nb = Button(\":cta.label\")"}], "prompt": "hero with cta"} +{"group_id": "mm_01", "modes": [{"canonical_fingerprint": "8c94a6a1e407ac005e58301670afa9aaccd0c2b1a04552693289f51a5a48ddf3", "mode_index": 0, "placeholders": [":body"], "source": "root = Stack([t], \"column\")\nt = TextContent(\":body\")"}, {"canonical_fingerprint": "8b2366208036fe324c2584d98e62cd697d1e0c2e1d2fc4461f19788d6510d691", "mode_index": 1, "placeholders": [":body"], "source": "root = Stack([c], \"column\")\nc = Card([t], \"column\")\nt = TextContent(\":body\")"}], "prompt": "simple text"} +{"group_id": "mm_02", "modes": [{"canonical_fingerprint": "4d0cafaed2d1d426605275844c7ffe9cbccb8c017cf18b7c4bff0932199f6fc8", "mode_index": 0, "placeholders": [":form.email", ":form.submit"], "source": "root = Form([i, s], \"column\")\ni = TextContent(\":form.email\")\ns = Button(\":form.submit\")"}, {"canonical_fingerprint": "29b4a54a4aa6b58ea3cd3ef25a8b282da7a08459de3959cd97443f1eec8e5d2d", "mode_index": 1, "placeholders": [":form.email", ":form.submit"], "source": "root = Form([c], \"column\")\nc = Card([i, s], \"column\")\ni = TextContent(\":form.email\")\ns = Button(\":form.submit\")"}], "prompt": "form with email and submit"} +{"group_id": "mm_03", "modes": [{"canonical_fingerprint": "5a798e31a8ce98a2a1a616a7e61531c632036bfc150d151bd775edc8903bff5b", "mode_index": 0, "placeholders": [":img.src", ":img.caption"], "source": "root = Stack([c], \"column\")\nc = Card([im, t], \"column\")\nim = Image(\":img.src\")\nt = TextContent(\":img.caption\")"}, {"canonical_fingerprint": "97b12398d55fd0836abcb35011122985efeb8dfbaf47d9d8d211f331aa598952", "mode_index": 1, "placeholders": [":img.src", ":img.caption"], "source": "root = Card([im, t], \"column\")\nim = Image(\":img.src\")\nt = TextContent(\":img.caption\")"}], "prompt": "card with image"} +{"group_id": "mm_04", "modes": [{"canonical_fingerprint": "b8528408c1ecc2292846532b492e323dec85034f4d98c445a1bfcb22d5120f58", "mode_index": 0, "placeholders": [":title", ":subtitle"], "source": "root = Stack([a, b], \"column\")\na = TextContent(\":title\")\nb = TextContent(\":subtitle\")"}, {"canonical_fingerprint": "9a2d5f9922a1612e18ac88aaed8a1c807f9d61f214ffab1787ede721e85db843", "mode_index": 1, "placeholders": [":title", ":subtitle"], "source": "root = Stack([c], \"column\")\nc = Card([a, b], \"column\")\na = TextContent(\":title\")\nb = TextContent(\":subtitle\")"}], "prompt": "two texts"} +{"group_id": "mm_05", "modes": [{"canonical_fingerprint": "13b7964bf2252bdbe82c5ce990ada9242baffb0f3b0b269be614ecefeedf6dd9", "mode_index": 0, "placeholders": [":cta"], "source": "root = Stack([b], \"column\")\nb = Button(\":cta\")"}, {"canonical_fingerprint": "1b9cd54f0cf1b9a5bac1f2c7bc8120ab78bccc0f7dc7c247705e065e7591c89a", "mode_index": 1, "placeholders": [":cta"], "source": "root = Card([b], \"column\")\nb = Button(\":cta\")"}], "prompt": "button only"} +{"group_id": "mm_06", "modes": [{"canonical_fingerprint": "de0096e262a8a20e11bb2338b2b99f3b38fca2034d657bf90c39ee7287d7d958", "mode_index": 0, "placeholders": [":card.body"], "source": "root = Stack([c], \"column\")\nc = Card([t], \"column\")\nt = TextContent(\":card.body\")"}, {"canonical_fingerprint": "54f7897efa57c43a2166b33229c3b1a8f043c020d84d78ef193c2aa4bfc0fd88", "mode_index": 1, "placeholders": [":card.body"], "source": "root = Stack([c], \"column\")\nc = Card([inner], \"column\")\ninner = Card([t], \"column\")\nt = TextContent(\":card.body\")"}], "prompt": "nested card"} +{"group_id": "mm_07", "modes": [{"canonical_fingerprint": "d60cd88bf91091b9ffc0087cf970ba229baa342ba8b955b32c4a906da8e881a3", "mode_index": 0, "placeholders": [":q"], "source": "root = Form([i], \"column\")\ni = TextContent(\":q\")"}, {"canonical_fingerprint": "c7f2a0d70546000767905aa498e7eaea65cdbfe66095ad6e9d93eab629e01969", "mode_index": 1, "placeholders": [":q"], "source": "root = Form([c], \"column\")\nc = Card([i], \"column\")\ni = TextContent(\":q\")"}], "prompt": "form single field"} diff --git a/src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json b/src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json new file mode 100644 index 000000000..32208cfff --- /dev/null +++ b/src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json @@ -0,0 +1,19 @@ +{ + "dataset_schema": "slm314-multimode-dataset/v1", + "manifest_sha256": "10d1edc939cc74c159858bf423aa3b457d691526efc41a125c50c5e97c7d42f9", + "modes_per_prompt": [ + 2 + ], + "n_modes": 16, + "n_prompts": 8, + "row_sha256": [ + "17bd182f4bddf979caabdf1248fe325418a4047575600ae6d10cad1c47314774", + "f6ac316055c35daefbdb864879ff504f269326d8d0bde64fc585b14d8be28882", + "015568fa00c3a12f895aeb58af0b85cb799e28f2c315d42fe02bfb573dbca89e", + "b22959ee91c136cbb9814a20ef9b4cf579924bab79c7c2f15fc3497e6ac9df62", + "35bd9d295405696d90b4748c671460a2db892552d824760b1a170ee56b436347", + "aad565646ad04affe14603d1eb0de57fed8d189e1ed2c7c360bf72ed8c78a5f6", + "708f07f6793b1832d3bee49b1f62fa2ab7ef3157b8de5f938c631888fa056b99", + "4b29cac7b71e9aff55a1914f674bc5f54656e01210bdcb6db714f34abccd73fd" + ] +} diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 1ec0d23fb..0edc43962 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -7859,6 +7859,26 @@ "note": "initial registration; SLM-312 LAR2-04 state sources: offline seedward oracle-guided valid intermediates + immutable content-addressed on-policy beam-trajectory snapshots labeled with the SLM-308 distance oracle and SLM-305 legal edits, fail-closed AST-fingerprint leak guards, matched {gold_only, seedward, on_policy, mixed} mixture arms with preregistered thresholds" } ] + }, + "harness.experiments.slm314_winner_take_all": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm314_winner_take_all.py", + "scripts/run_slm314_winner_take_all.py", + "tests/test_harnesses/experiments/test_slm314_winner_take_all.py", + "src/slm_training/resources/data/slm314_multimode/modes.jsonl", + "src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json", + "docs/design/iter-slm314-winner-take-all-20260724.json", + "docs/design/iter-slm314-winner-take-all-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-314 LAR2-05 winner-take-all (MCL): frozen 8-prompt x 2-mode verifier-accepted canonical-AST dataset (alpha-invariant fingerprints + sha256 manifest), min-loss-mode loss with preregistered 0.1 coverage/floor term, deterministic two-mode synthetic collapse-vs-retain proof, matched {single_gold, multi_gold, wta} arms" + } + ] } } } diff --git a/tests/test_harnesses/experiments/test_slm314_winner_take_all.py b/tests/test_harnesses/experiments/test_slm314_winner_take_all.py new file mode 100644 index 000000000..82be75e7c --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm314_winner_take_all.py @@ -0,0 +1,244 @@ +"""SLM-314 (LAR2-05): multi-mode dataset freeze, WTA loss, synthetic fixture.""" + +from __future__ import annotations + +import json + +import pytest + +# Tiny CPU fixture models: torch thread thrash dominates otherwise. +import torch + +torch.set_num_threads(1) + +from slm_training.dsl.schema import ExampleRecord +from slm_training.harnesses.experiments.slm314_winner_take_all import ( + DEFAULT_RESOURCE, + FLOOR_EPSILON, + FreezeIntegrityError, + PromptGroupV1, + build_multimode_dataset, + dataset_manifest, + freeze_dataset, + mode_supervision_loss, + read_frozen_dataset, + run_synthetic_two_mode, + select_winner, + single_gold_groups, + synthetic_assertions, + winner_take_all_loss, + _fingerprint_source, +) +from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, +) + +_SMALL_CONFIG = { + "context_backend": "scratch", + "seed": 0, + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 2, + "max_search_steps": 4, + "beam_width": 2, + "expand_per_state": 2, +} + +MODE_A = 'root = Stack([t], "column")\nt = TextContent(":body")' +MODE_B = 'root = Stack([c], "column")\nc = Card([t], "column")\nt = TextContent(":body")' + + +def _group() -> PromptGroupV1: + return next(g for g in build_multimode_dataset() if g.group_id == "mm_01") + + +def _model(records: list[ExampleRecord]) -> TreeEditDiffusionModel: + torch.manual_seed(0) + return TreeEditDiffusionModel.from_records( + records, config=TreeEditDiffusionConfig(**_SMALL_CONFIG), device="cpu" + ) + + +# --- mode identity via canonical fingerprints ------------------------------- + + +def test_fingerprint_alpha_invariant_same_ast() -> None: + """Two serializations of the same AST (renamed statements, reordered + declarations) share one canonical fingerprint.""" + renamed = 'root = Stack([x1], "column")\nx1 = TextContent(":body")' + reordered = 't = TextContent(":body")\nroot = Stack([t], "column")' + assert _fingerprint_source(MODE_A) == _fingerprint_source(renamed) + assert _fingerprint_source(MODE_A) == _fingerprint_source(reordered) + + +def test_fingerprint_distinct_for_different_asts() -> None: + assert _fingerprint_source(MODE_A) != _fingerprint_source(MODE_B) + + +def test_fingerprint_requires_verifier_acceptance() -> None: + with pytest.raises(ValueError): + _fingerprint_source("root = NotAComponent([t])") + + +# --- frozen dataset ---------------------------------------------------------- + + +def test_dataset_every_prompt_has_two_distinct_valid_modes() -> None: + groups = build_multimode_dataset() + assert len(groups) == 8 + for group in groups: + assert len(group.modes) >= 2 + fps = [m.canonical_fingerprint for m in group.modes] + assert len(set(fps)) == len(fps) + for mode in group.modes: + # fingerprint identity is recomputed through the real parser + assert _fingerprint_source(mode.source) == mode.canonical_fingerprint + + +def test_committed_resource_matches_builder_and_manifest() -> None: + frozen = read_frozen_dataset(DEFAULT_RESOURCE) + live = build_multimode_dataset() + assert [g.to_dict() for g in frozen] == [g.to_dict() for g in live] + assert dataset_manifest(frozen)["n_modes"] == 16 + + +def test_freeze_roundtrip_and_tamper_detection(tmp_path) -> None: + path = tmp_path / "modes.jsonl" + groups = build_multimode_dataset() + manifest = freeze_dataset(groups, path) + assert dataset_manifest(read_frozen_dataset(path)) == manifest + # Tamper with one mode source: fail-closed on reload. + rows = [json.loads(line) for line in path.read_text().splitlines()] + rows[0]["modes"][0]["source"] = MODE_B + path.write_text("\n".join(json.dumps(r, sort_keys=True) for r in rows) + "\n") + with pytest.raises(FreezeIntegrityError): + read_frozen_dataset(path) + + +def test_single_gold_groups_deterministic() -> None: + groups = build_multimode_dataset() + reduced = single_gold_groups(groups) + assert all(len(g.modes) == 1 for g in reduced) + for full, one in zip(groups, reduced): + expected = min(m.canonical_fingerprint for m in full.modes) + assert one.modes[0].canonical_fingerprint == expected + + +# --- WTA loss ---------------------------------------------------------------- + + +def test_select_winner_min_loss_tie_breaks_lowest() -> None: + assert select_winner([2.0, 1.0, 3.0]) == 1 + assert select_winner([1.0, 1.0]) == 0 + with pytest.raises(ValueError): + select_winner([]) + + +def test_wta_selects_min_loss_mode_and_records_telemetry() -> None: + group = _group() + records = [r for g in build_multimode_dataset() for r in g.records()] + model = _model(records) + loss, telemetry = winner_take_all_loss(model, group, seed=0) + losses = [m["loss"] for m in telemetry["per_mode"]] + winner = telemetry["winner_index"] + assert winner == select_winner(losses) + assert telemetry["winner_fingerprint"] == ( + group.modes[winner].canonical_fingerprint + ) + assert sum(1 for m in telemetry["per_mode"] if m["selected"]) == 1 + # total = winner + eps * mean(losers) + losers = [x for i, x in enumerate(losses) if i != winner] + expected = losses[winner] + FLOOR_EPSILON * (sum(losers) / len(losers)) + assert telemetry["total_loss"] == pytest.approx(expected, rel=1e-5) + assert float(loss.detach()) == pytest.approx(expected, rel=1e-5) + + +def test_wta_floor_term_backpropagates_into_losing_mode() -> None: + """The coverage/floor term must constrain the losing mode: gradient flows + into both modes' losses (winner fully, loser with the epsilon weight).""" + la = torch.tensor(2.0, requires_grad=True) + lb = torch.tensor(3.0, requires_grad=True) + losses = [la, lb] + winner = select_winner([2.0, 3.0]) + losers = [x for i, x in enumerate(losses) if i != winner] + total = losses[winner] + FLOOR_EPSILON * (sum(losers) / len(losers)) + total.backward() + assert la.grad.item() == pytest.approx(1.0) + assert lb.grad.item() == pytest.approx(FLOOR_EPSILON) + + +def test_mode_supervision_loss_deterministic_and_rng_restored() -> None: + group = _group() + records = [r for g in build_multimode_dataset() for r in g.records()] + model = _model(records) + record = group.records()[0] + rng_state_before = model._rng.getstate() + l1 = float(mode_supervision_loss(model, record, seed=123).detach()) + l2 = float(mode_supervision_loss(model, record, seed=123).detach()) + assert l1 == pytest.approx(l2) + assert model._rng.getstate() == rng_state_before + + +def test_winner_take_all_loss_deterministic() -> None: + group = _group() + records = [r for g in build_multimode_dataset() for r in g.records()] + m1 = _model(records) + m2 = _model(records) + l1, t1 = winner_take_all_loss(m1, group, seed=7) + l2, t2 = winner_take_all_loss(m2, group, seed=7) + assert t1 == t2 + assert float(l1.detach()) == pytest.approx(float(l2.detach())) + + +# --- synthetic two-mode collapse-vs-retain fixture --------------------------- + + +def test_synthetic_single_gold_collapses_wta_retains() -> None: + result = run_synthetic_two_mode() + checks = synthetic_assertions(result) + assert all(checks.values()), checks + # Explicit collapse/retain numbers (preregistered thresholds). + assert result["single_gold"]["p_mode_b"] < 0.002 + assert result["wta"]["p_mode_b"] >= 0.004 + assert result["wta"]["p_mode_a"] > result["wta"]["p_mode_b"] + # Averaging arm puts >=30% mass on invalid hybrids. + assert result["multi_gold"]["p_invalid_hybrid"] >= 0.3 + + +def test_synthetic_deterministic() -> None: + assert run_synthetic_two_mode() == run_synthetic_two_mode() + + +def test_synthetic_floor_controls_retained_mass() -> None: + """Larger floor epsilon -> more losing-mode mass retained (monotone).""" + low = run_synthetic_two_mode(floor_epsilon=0.05) + high = run_synthetic_two_mode(floor_epsilon=0.3) + assert high["wta"]["p_mode_b"] > low["wta"]["p_mode_b"] + + +# --- arm isolation ------------------------------------------------------------ + + +def test_arm_isolation_identical_budgets() -> None: + """The CLI's train_arm gives every arm the same model init, step count, + and per-step prompt coverage; only the loss weighting differs.""" + from scripts.run_slm314_winner_take_all import ARMS, FIXTURE_CONFIG_OVERRIDES + + assert ARMS == ("single_gold", "multi_gold", "wta") + # Same declared fixture config object for all arms (no per-arm overrides). + assert set(FIXTURE_CONFIG_OVERRIDES) >= {"d_model", "denoiser_layers"} + groups = build_multimode_dataset() + records = [r for g in groups for r in g.records()] + torch.manual_seed(0) + m1 = TreeEditDiffusionModel.from_records( + records, config=TreeEditDiffusionConfig(**_SMALL_CONFIG), device="cpu" + ) + torch.manual_seed(0) + m2 = TreeEditDiffusionModel.from_records( + records, config=TreeEditDiffusionConfig(**_SMALL_CONFIG), device="cpu" + ) + s1 = {k: v.detach().clone() for k, v in m1.state_dict().items()} + s2 = m2.state_dict() + assert all(torch.equal(s1[k], s2[k]) for k in s1) From 48e5cadc13f9e6e431491da615badb4a0614c17a Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:24:37 -0500 Subject: [PATCH 09/18] SLM-317: do-no-harm AR->repair hybrid + advancement screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Metamorphic invariance (alpha-rename, statement reorder, formatting, AST roundtrip) tested before model evaluation. - Hard do-no-harm commit rule: COMMIT on deterministic hard-evidence improvement or calibrated soft improvement without hard regression; RETAIN on regression/no-improvement; ABSTAIN on missing candidate; source + candidate + exact reason durable per example; oracle commit selector as non-promotable upper bound. - Matched arms {ar_only, repair_only, ar_repair_historical, ar_repair_improved, oracle_commit} on the frozen SLM-155 decision corpus with paired per-example outcomes and explicit invalid-over-valid counts per arm. - Gates: Safety PASS (iov=0), Value FAIL (0/16 improvements, Wilson [0.0, 0.194] vs 0.05), Reachability PASS. Disposition: inconclusive — LAR3 remains open but NOT advanced; powered rerun required. - 20 new tests; slm317 component v1. --- .../iter-slm317-repair-hybrid-20260724.json | 899 ++++++++++++++++++ .../iter-slm317-repair-hybrid-20260724.md | 53 ++ scripts/run_slm317_repair_hybrid.py | 631 ++++++++++++ .../experiments/slm317_repair_hybrid.py | 446 +++++++++ src/slm_training/resources/versions.json | 18 + .../experiments/test_slm317_repair_hybrid.py | 263 +++++ 6 files changed, 2310 insertions(+) create mode 100644 docs/design/iter-slm317-repair-hybrid-20260724.json create mode 100644 docs/design/iter-slm317-repair-hybrid-20260724.md create mode 100644 scripts/run_slm317_repair_hybrid.py create mode 100644 src/slm_training/harnesses/experiments/slm317_repair_hybrid.py create mode 100644 tests/test_harnesses/experiments/test_slm317_repair_hybrid.py diff --git a/docs/design/iter-slm317-repair-hybrid-20260724.json b/docs/design/iter-slm317-repair-hybrid-20260724.json new file mode 100644 index 000000000..eddd68faf --- /dev/null +++ b/docs/design/iter-slm317-repair-hybrid-20260724.json @@ -0,0 +1,899 @@ +{ + "experiment": "slm317-repair-hybrid", + "issue": "SLM-317", + "question": "Does a do-no-harm AR\u2192repair hybrid (deterministic commit rule) improve paired hard-valid outcomes over AR only without ever damaging a valid AR program?", + "preregistered": { + "min_useful_effect": 0.05, + "safety_rule": "invalid-over-valid count of ar_repair_improved vs ar_only on the frozen safety set (all eval examples) must be exactly 0", + "value_rule": "paired hard-valid improvement of ar_repair_improved over ar_only: Wilson 95% lower bound of improvements/n_paired must exceed 0.05 with zero damages", + "disposition_rule": "repair_positive iff safety AND value AND reachability gates pass; repair_negative iff safety fails OR the value Wilson UPPER bound is below the minimum useful effect (effect ruled out); else inconclusive", + "historical_deviation": "pre-SLM-305 4-action edit space is not reproducible on this branch (edit space extended in place, no legacy-subset knob); the historical arm keeps mutation_count value labels + legacy STOP-slot accounting only" + }, + "arms": [ + "ar_only", + "repair_only", + "ar_repair_historical", + "ar_repair_improved", + "oracle_commit" + ], + "promotable_arms": [ + "ar_only", + "ar_repair_historical", + "ar_repair_improved", + "repair_only" + ], + "config": { + "n_train_decisions": 16, + "n_eval_decisions": 8, + "seeds": [ + 0, + 1 + ], + "steps": 6, + "lr": 0.003, + "scorer_steps": 20, + "fixture_config_overrides": { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 3, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3 + }, + "torch_num_threads": 1 + }, + "arm_summaries": { + "ar_only": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "repair_only": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "ar_repair_historical": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "ar_repair_improved": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "oracle_commit": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + } + }, + "paired_vs_ar_only": { + "repair_only": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + }, + "ar_repair_historical": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + }, + "ar_repair_improved": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + }, + "oracle_commit": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + } + }, + "invalid_over_valid_per_arm": { + "repair_only": 0, + "ar_repair_historical": 0, + "ar_repair_improved": 0, + "oracle_commit": 0 + }, + "commit_reason_counts": { + "ar_repair_historical": { + "hard_regression": 16 + }, + "ar_repair_improved": { + "no_improvement": 16 + }, + "oracle_commit": { + "oracle_no_gain": 16 + } + }, + "gates": { + "safety": { + "rule": "invalid-over-valid count of ar_repair_improved vs ar_only on the frozen safety set (all eval examples) must be exactly 0", + "invalid_over_valid": 0, + "pass": true + }, + "value": { + "rule": "paired hard-valid improvement of ar_repair_improved over ar_only: Wilson 95% lower bound of improvements/n_paired must exceed 0.05 with zero damages", + "improvements": 0, + "n_paired": 16, + "wilson": { + "n": 16, + "estimate": 0.0, + "low": 0.0, + "high": 0.19360768053443644, + "confidence_level": 0.95 + }, + "min_useful_effect": 0.05, + "pass": false + }, + "reachability": { + "artifacts": { + "docs/design/iter-slm299-edit-reachability-20260724.json": true, + "docs/design/iter-slm299-edit-reachability-20260724.md": true, + "docs/design/iter-slm291-evidence-bundles-20260724.json": true, + "docs/design/iter-slm291-evidence-bundles-20260724.md": true + }, + "pass": true + } + }, + "wilson_damage": { + "n": 16, + "estimate": 0.0, + "low": 0.0, + "high": 0.19360768053443644, + "confidence_level": 0.95 + }, + "oracle_upper_bound": { + "valid_rate": 1.0, + "sanity": true + }, + "per_example_outcomes": [ + { + "record_id": "decision-0", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-0", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-1", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-1", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-2", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-2", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-3", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-3", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-4", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-4", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-5", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-5", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-6", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-6", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-7", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-7", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-0", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-0", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-1", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-1", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-2", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-2", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-3", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-3", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-4", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-4", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-5", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-5", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-6", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-6", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-7", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-7", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + } + ], + "decisions_path": "outputs/slm317/commit_decisions.jsonl", + "disposition": "inconclusive", + "lar3_statement": "LAR3 (learned repair integration) remains OPEN but NOT advanced: the screen neither cleared the value gate nor ruled the effect out at fixture power; a powered rerun is required before LAR3 can open or close.", + "wall_seconds": 67.08616952600278, + "honesty": "Fixture-scale wiring screen: the frozen corpus is the SLM-155 synthetic decision fixture (n eval decisions below any ship-gate prompt count), models are tiny CPU fixtures trained for a handful of steps, and no ship-gate claim is made. The commit rule is deterministic and metamorphism-invariant by construction; the screen measures the mechanism, not production quality. The historical arm's pre-SLM-305 4-action space is declared non-reproducible on this branch (see preregistered deviation).", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "f6b173105eb2f3e2e5fa2b5fea7e3af452cc2b83", + "code_dirty": true, + "components": { + "harness.experiments.slm317_repair_hybrid": "v1", + "harness.experiments.slm155_factorization_comparison": "v2", + "harness.experiments.slm299_edit_reachability": "v4" + }, + "stamped_at": "2026-07-25T00:11:13.905812+00:00" + } +} diff --git a/docs/design/iter-slm317-repair-hybrid-20260724.md b/docs/design/iter-slm317-repair-hybrid-20260724.md new file mode 100644 index 000000000..e7e2d5e05 --- /dev/null +++ b/docs/design/iter-slm317-repair-hybrid-20260724.md @@ -0,0 +1,53 @@ +# SLM-317 (LAR2-06): do-no-harm AR→repair hybrid screen + +**Disposition: `inconclusive`** — LAR3 (learned repair integration) remains OPEN but NOT advanced: the screen neither cleared the value gate nor ruled the effect out at fixture power; a powered rerun is required before LAR3 can open or close. + +Fixture-scale mechanism screen; **not a ship claim**. + +## Preregistered (locked before results) + +- safety: invalid-over-valid count of ar_repair_improved vs ar_only on the frozen safety set (all eval examples) must be exactly 0 +- value: paired hard-valid improvement of ar_repair_improved over ar_only: Wilson 95% lower bound of improvements/n_paired must exceed 0.05 with zero damages +- disposition: repair_positive iff safety AND value AND reachability gates pass; repair_negative iff safety fails OR the value Wilson UPPER bound is below the minimum useful effect (effect ruled out); else inconclusive +- declared deviation: pre-SLM-305 4-action edit space is not reproducible on this branch (edit space extended in place, no legacy-subset knob); the historical arm keeps mutation_count value labels + legacy STOP-slot accounting only + +## Arms (matched examples + seeds + budgets) + +| arm | n | hard-valid rate | mean hard rank | invalid-over-valid vs AR | +| --- | --- | --- | --- | --- | +| ar_only | 16 | 1.000 | 3.000 | — | +| repair_only | 16 | 1.000 | 3.000 | 0 | +| ar_repair_historical | 16 | 1.000 | 3.000 | 0 | +| ar_repair_improved | 16 | 1.000 | 3.000 | 0 | +| oracle_commit | 16 | 1.000 | 3.000 | 0 | + +## Paired outcomes vs `ar_only` (per-example, nothing aggregated away) + +| arm | both valid | AR valid, arm invalid | AR invalid, arm valid | both invalid | unpaired | +| --- | --- | --- | --- | --- | --- | +| repair_only | 16 | 0 | 0 | 0 | 0 | +| ar_repair_historical | 16 | 0 | 0 | 0 | 0 | +| ar_repair_improved | 16 | 0 | 0 | 0 | 0 | +| oracle_commit | 16 | 0 | 0 | 0 | 0 | + +## Advancement gates + +- **Safety**: invalid-over-valid = 0 (must be 0) → PASS +- **Value**: improvements 0/16, Wilson 95% [0.0, 0.19360768053443644] vs minimum useful effect 0.05 → FAIL +- **Reachability/provenance**: slm299/291 artifacts present → PASS + +## Commit reasons + +| arm | reason | count | +| --- | --- | --- | +| ar_repair_historical | hard_regression | 16 | +| ar_repair_improved | no_improvement | 16 | +| oracle_commit | oracle_no_gain | 16 | + +Oracle commit upper bound hard-valid rate: 1.000 (sanity ≥ improved hybrid: True). + +Durable per-example commit decisions: `outputs/slm317/commit_decisions.jsonl`. + +## Honesty + +Fixture-scale wiring screen: the frozen corpus is the SLM-155 synthetic decision fixture (n eval decisions below any ship-gate prompt count), models are tiny CPU fixtures trained for a handful of steps, and no ship-gate claim is made. The commit rule is deterministic and metamorphism-invariant by construction; the screen measures the mechanism, not production quality. The historical arm's pre-SLM-305 4-action space is declared non-reproducible on this branch (see preregistered deviation). diff --git a/scripts/run_slm317_repair_hybrid.py b/scripts/run_slm317_repair_hybrid.py new file mode 100644 index 000000000..932216a88 --- /dev/null +++ b/scripts/run_slm317_repair_hybrid.py @@ -0,0 +1,631 @@ +#!/usr/bin/env python3 +"""SLM-317 (LAR2-06): valid-state repair advancement screen (AR→repair hybrid). + +Runs the preregistered do-no-harm screen over matched arms on the frozen +SLM-155 fixture decision corpus (identical examples + seeds + budgets): + +- ``ar_only`` — the SLM-155 AR legal-action path; final = AR program. +- ``repair_only`` — X22 value-guided beam decode from the minimal seed. +- ``ar_repair_historical`` — AR source → repair decode under the pre-LAR2 + config (``mutation_count`` value labels, legacy STOP-slot accounting). The + pre-SLM-305 4-action edit space is NOT reproducible on this branch (the + space was extended in place; no legacy-subset knob exists) — declared + deviation, the arm keeps the historical value/stop config only. +- ``ar_repair_improved`` — AR source → repair decode under this branch's + config (``bounded_distance`` oracle value labels + corrected STOP + accounting), then the do-no-harm commit rule with the policy value head as + the calibrated learned score. +- ``oracle_commit`` — commits exactly when the candidate's hard evidence is + strictly better (non-promotable upper bound). + +Every final is scored on the LAR1-02 unified ladder (official parse → output +contract → meaningful_program_v1). Paired per-example outcomes key on +(record_id, seed); invalid-over-valid counts are reported per arm and never +aggregated away. Advancement gates: Safety = 0 invalid-over-valid on the +frozen safety set; Value = paired improvement over AR with Wilson lower bound +above the preregistered minimum useful effect (0.05); Reachability/provenance += LAR2 artifacts present (slm299 reachability, slm291 evidence bundles). +Exactly one disposition: repair_positive | repair_negative | inconclusive, +with an explicit LAR3 open/close statement. + +Writes ``docs/design/iter-slm317-repair-hybrid-20260724.{json,md}`` and +durable commit decisions under ``outputs/slm317/``. + +Example: + python -m scripts.run_slm317_repair_hybrid --steps 8 +""" + +from __future__ import annotations + +import argparse +import json +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from slm_training.evals.power_protocol import wilson_interval +from slm_training.harnesses.experiments.slm155_factorization_comparison import ( + _program_from_actions, + paired_validity_table, +) +from slm_training.harnesses.experiments.slm317_repair_hybrid import ( + DISPOSITION_INCONCLUSIVE, + DISPOSITION_NEGATIVE, + DISPOSITION_POSITIVE, + EXPERIMENT_ID, + CommitDecision, + commit, + hard_evidence, + oracle_commit, + repair_decode, + state_value, +) +from slm_training.models.legal_action_scorer import ( + LegalActionScorerConfig, + make_fixture_decisions, + train_fixture_scorer, +) +from slm_training.versioning import build_version_stamp + +DEFAULT_JSON_OUT = Path("docs/design/iter-slm317-repair-hybrid-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm317-repair-hybrid-20260724.md") +DEFAULT_DECISIONS_OUT = Path("outputs/slm317/commit_decisions.jsonl") + +ARMS = ( + "ar_only", + "repair_only", + "ar_repair_historical", + "ar_repair_improved", + "oracle_commit", +) +PROMOTABLE_ARMS = {"ar_only", "repair_only", "ar_repair_historical", "ar_repair_improved"} + +# LAR2 reachability/provenance artifacts this screen builds on. +REACHABILITY_ARTIFACTS = ( + "docs/design/iter-slm299-edit-reachability-20260724.json", + "docs/design/iter-slm299-edit-reachability-20260724.md", + "docs/design/iter-slm291-evidence-bundles-20260724.json", + "docs/design/iter-slm291-evidence-bundles-20260724.md", +) + +# PREREGISTERED (locked before any run; deviations append-only/exploratory). +PREREGISTERED = { + "min_useful_effect": 0.05, + "safety_rule": ( + "invalid-over-valid count of ar_repair_improved vs ar_only on the " + "frozen safety set (all eval examples) must be exactly 0" + ), + "value_rule": ( + "paired hard-valid improvement of ar_repair_improved over ar_only: " + "Wilson 95% lower bound of improvements/n_paired must exceed 0.05 " + "with zero damages" + ), + "disposition_rule": ( + "repair_positive iff safety AND value AND reachability gates pass; " + "repair_negative iff safety fails OR the value Wilson UPPER bound is " + "below the minimum useful effect (effect ruled out); else inconclusive" + ), + "historical_deviation": ( + "pre-SLM-305 4-action edit space is not reproducible on this branch " + "(edit space extended in place, no legacy-subset knob); the " + "historical arm keeps mutation_count value labels + legacy STOP-slot " + "accounting only" + ), +} + +FIXTURE_CONFIG_OVERRIDES = { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 3, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3, +} + + +@dataclass(frozen=True) +class ArmOutcome: + """Paired per-example final outcome for one arm (duck-types slm155's).""" + + record_id: str + arm_id: str + seed: int + final_source: str + v1_verdict: bool + hard_rank: int + commit_reason: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "record_id": self.record_id, + "arm_id": self.arm_id, + "seed": self.seed, + "final_source": self.final_source, + "v1_verdict": self.v1_verdict, + "hard_rank": self.hard_rank, + "commit_reason": self.commit_reason, + } + + +def _inventory_for(source: str) -> list[str]: + import re + + slots = sorted(set(re.findall(r":[A-Za-z][\w.]*", source))) + return slots or [":content"] + + +def _prompt_for(decision_id: str) -> str: + return f"fixture decision {decision_id}: render the accepted content" + + +def _build_model(records, *, seed: int, value_label_mode: str, stop_accounting: str): + import torch + + from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + ) + + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + seed=seed, + value_label_mode=value_label_mode, + stop_slot_accounting=stop_accounting, + context_backend="scratch", + **FIXTURE_CONFIG_OVERRIDES, + ) + return TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + + +def _train(model, records, *, steps: int, lr: float): + import torch + + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=lr) + model.train() + for _ in range(steps): + loss = model.training_loss(records) + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +def run_screen( + *, + n_train: int, + n_eval: int, + seeds: tuple[int, ...], + steps: int, + lr: float, + scorer_steps: int, + decisions_out: Path, +) -> dict[str, Any]: + import torch + + from slm_training.dsl.schema import ExampleRecord + + torch.set_num_threads(1) # tiny CPU fixture; identical for every arm + start = time.perf_counter() + + train_decisions = make_fixture_decisions(n=n_train, seed=0) + eval_decisions = make_fixture_decisions(n=n_eval, seed=1) + scorer_result = train_fixture_scorer( + train_decisions, + config=LegalActionScorerConfig(variant="mlp", seed=0), + steps=scorer_steps, + lr=0.05, + ) + scorer = scorer_result["scorer"] + + def ar_source_for(decision) -> str: + scores = scorer.score( + decision.context_features, + decision.state_features, + decision.legal_actions, + plan_features=decision.plan_features, + plan_action_features=decision.plan_action_features, + pack_id=decision.pack_id, + ) + chosen = scorer.decode(scores, decision.legal_actions).action_identity + return _program_from_actions([chosen] if chosen else []) + + train_records = [ + ExampleRecord( + id=d.decision_id, + prompt=_prompt_for(d.decision_id), + openui=_program_from_actions( + [d.accepted_action_ids[0]] if d.accepted_action_ids else [] + ), + placeholders=_inventory_for( + _program_from_actions( + [d.accepted_action_ids[0]] if d.accepted_action_ids else [] + ) + ), + ) + for d in train_decisions + ] + + model_hist = _train( + _build_model( + train_records, + seed=0, + value_label_mode="mutation_count", + stop_accounting="legacy", + ), + train_records, + steps=steps, + lr=lr, + ) + model_imp = _train( + _build_model( + train_records, + seed=0, + value_label_mode="bounded_distance", + stop_accounting="corrected", + ), + train_records, + steps=steps, + lr=lr, + ) + + outcomes: list[ArmOutcome] = [] + commit_rows: list[CommitDecision] = [] + + def record(arm: str, record_id: str, seed: int, final: str, reason: str | None): + ev = hard_evidence(final) + outcomes.append( + ArmOutcome( + record_id=record_id, + arm_id=arm, + seed=seed, + final_source=final, + v1_verdict=ev.v1_verdict, + hard_rank=ev.rank, + commit_reason=reason, + ) + ) + + for seed in seeds: + for decision in eval_decisions: + rid = decision.decision_id + prompt = _prompt_for(rid) + ar_source = ar_source_for(decision) + inventory = _inventory_for(ar_source) + + # Arm 1: AR only. + record("ar_only", rid, seed, ar_source, None) + + # Arm 2: repair-only from the minimal seed (improved model decode). + ctx, ctx_pad = model_imp._encode_context( + [model_imp._format_context(prompt, slot_contract=inventory)] + ) + seed_final, _ = model_imp._decode_one(ctx, ctx_pad, inventory) + record("repair_only", rid, seed, seed_final, None) + + # Arm 3: AR → historical repair (pre-LAR2 value/stop config). + cand_hist, _ = repair_decode(model_hist, ar_source, inventory, prompt) + decision_hist = commit( + ar_source, + cand_hist, + evidence={ + "learned_score_source": state_value( + model_hist, ar_source, inventory, prompt + ), + "learned_score_candidate": state_value( + model_hist, cand_hist, inventory, prompt + ), + }, + record_id=rid, + arm_id="ar_repair_historical", + seed=seed, + ) + commit_rows.append(decision_hist) + record( + "ar_repair_historical", rid, seed, decision_hist.final, decision_hist.reason + ) + + # Arm 4: AR → improved repair (this branch's config). + cand_imp, _ = repair_decode(model_imp, ar_source, inventory, prompt) + decision_imp = commit( + ar_source, + cand_imp, + evidence={ + "learned_score_source": state_value( + model_imp, ar_source, inventory, prompt + ), + "learned_score_candidate": state_value( + model_imp, cand_imp, inventory, prompt + ), + }, + record_id=rid, + arm_id="ar_repair_improved", + seed=seed, + ) + commit_rows.append(decision_imp) + record( + "ar_repair_improved", rid, seed, decision_imp.final, decision_imp.reason + ) + + # Arm 5: oracle commit selector (non-promotable upper bound). + decision_oracle = oracle_commit( + ar_source, cand_imp, record_id=rid, arm_id="oracle_commit", seed=seed + ) + commit_rows.append(decision_oracle) + record("oracle_commit", rid, seed, decision_oracle.final, decision_oracle.reason) + + decisions_out.parent.mkdir(parents=True, exist_ok=True) + with decisions_out.open("w", encoding="utf-8") as fh: + for row in commit_rows: + fh.write(json.dumps(row.to_dict(), sort_keys=True) + "\n") + + by_arm: dict[str, list[ArmOutcome]] = {arm: [] for arm in ARMS} + for outcome in outcomes: + by_arm[outcome.arm_id].append(outcome) + + ar_outcomes = by_arm["ar_only"] + paired: dict[str, dict[str, int]] = {} + for arm in ARMS: + if arm == "ar_only": + continue + paired[arm] = paired_validity_table(ar_outcomes, by_arm[arm]) + + def valid_rate(arm: str) -> float: + rows = by_arm[arm] + return sum(o.v1_verdict for o in rows) / max(len(rows), 1) + + n_paired = sum(paired["ar_repair_improved"].values()) - paired["ar_repair_improved"]["unpaired"] + improvements = paired["ar_repair_improved"]["a_invalid_b_valid"] + damages = paired["ar_repair_improved"]["a_valid_b_invalid"] + wilson = wilson_interval(improvements, max(n_paired, 0)) + wilson_damage = wilson_interval(damages, max(n_paired, 0)) + + reachability = {p: Path(p).exists() for p in REACHABILITY_ARTIFACTS} + gates = { + "safety": { + "rule": PREREGISTERED["safety_rule"], + "invalid_over_valid": damages, + "pass": damages == 0, + }, + "value": { + "rule": PREREGISTERED["value_rule"], + "improvements": improvements, + "n_paired": n_paired, + "wilson": wilson, + "min_useful_effect": PREREGISTERED["min_useful_effect"], + "pass": bool( + wilson["low"] is not None + and wilson["low"] > PREREGISTERED["min_useful_effect"] + and damages == 0 + ), + }, + "reachability": { + "artifacts": reachability, + "pass": all(reachability.values()), + }, + } + + safety_pass = gates["safety"]["pass"] + value_pass = gates["value"]["pass"] + reach_pass = gates["reachability"]["pass"] + if safety_pass and value_pass and reach_pass: + disposition = DISPOSITION_POSITIVE + elif not safety_pass or ( + wilson["high"] is not None + and wilson["high"] < PREREGISTERED["min_useful_effect"] + ): + disposition = DISPOSITION_NEGATIVE + else: + disposition = DISPOSITION_INCONCLUSIVE + + lar3_statement = { + DISPOSITION_POSITIVE: ( + "LAR3 (learned repair integration) is OPEN: the do-no-harm hybrid " + "cleared safety, value, and reachability gates on the frozen screen." + ), + DISPOSITION_NEGATIVE: ( + "LAR3 (learned repair integration) is CLOSED: repair damaged valid " + "AR programs or its paired value was ruled out below the minimum " + "useful effect on the frozen screen." + ), + DISPOSITION_INCONCLUSIVE: ( + "LAR3 (learned repair integration) remains OPEN but NOT advanced: " + "the screen neither cleared the value gate nor ruled the effect " + "out at fixture power; a powered rerun is required before LAR3 " + "can open or close." + ), + }[disposition] + + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-317", + "question": ( + "Does a do-no-harm AR→repair hybrid (deterministic commit rule) " + "improve paired hard-valid outcomes over AR only without ever " + "damaging a valid AR program?" + ), + "preregistered": PREREGISTERED, + "arms": list(ARMS), + "promotable_arms": sorted(PROMOTABLE_ARMS), + "config": { + "n_train_decisions": n_train, + "n_eval_decisions": n_eval, + "seeds": list(seeds), + "steps": steps, + "lr": lr, + "scorer_steps": scorer_steps, + "fixture_config_overrides": FIXTURE_CONFIG_OVERRIDES, + "torch_num_threads": 1, + }, + "arm_summaries": { + arm: { + "n": len(by_arm[arm]), + "valid_rate": valid_rate(arm), + "mean_hard_rank": ( + sum(o.hard_rank for o in by_arm[arm]) / max(len(by_arm[arm]), 1) + ), + } + for arm in ARMS + }, + "paired_vs_ar_only": paired, + "invalid_over_valid_per_arm": { + arm: paired[arm]["a_valid_b_invalid"] for arm in paired + }, + "commit_reason_counts": { + arm: { + reason: sum(1 for o in by_arm[arm] if o.commit_reason == reason) + for reason in sorted( + {o.commit_reason for o in by_arm[arm] if o.commit_reason} + ) + } + for arm in ("ar_repair_historical", "ar_repair_improved", "oracle_commit") + }, + "gates": gates, + "wilson_damage": wilson_damage, + "oracle_upper_bound": { + "valid_rate": valid_rate("oracle_commit"), + "sanity": valid_rate("oracle_commit") >= valid_rate("ar_repair_improved"), + }, + "per_example_outcomes": [o.to_dict() for o in outcomes], + "decisions_path": str(decisions_out), + "disposition": disposition, + "lar3_statement": lar3_statement, + "wall_seconds": time.perf_counter() - start, + "honesty": ( + "Fixture-scale wiring screen: the frozen corpus is the SLM-155 " + "synthetic decision fixture (n eval decisions below any ship-gate " + "prompt count), models are tiny CPU fixtures trained for a handful " + "of steps, and no ship-gate claim is made. The commit rule is " + "deterministic and metamorphism-invariant by construction; the " + "screen measures the mechanism, not production quality. The " + "historical arm's pre-SLM-305 4-action space is declared " + "non-reproducible on this branch (see preregistered deviation)." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm317_repair_hybrid", + "harness.experiments.slm155_factorization_comparison", + "harness.experiments.slm299_edit_reachability", + ) + return payload + + +def render_markdown(payload: dict) -> str: + def f(x: float) -> str: + return f"{x:.3f}" + + lines = [ + "# SLM-317 (LAR2-06): do-no-harm AR→repair hybrid screen", + "", + f"**Disposition: `{payload['disposition']}`** — {payload['lar3_statement']}", + "", + "Fixture-scale mechanism screen; **not a ship claim**.", + "", + "## Preregistered (locked before results)", + "", + f"- safety: {payload['preregistered']['safety_rule']}", + f"- value: {payload['preregistered']['value_rule']}", + f"- disposition: {payload['preregistered']['disposition_rule']}", + f"- declared deviation: {payload['preregistered']['historical_deviation']}", + "", + "## Arms (matched examples + seeds + budgets)", + "", + "| arm | n | hard-valid rate | mean hard rank | invalid-over-valid vs AR |", + "| --- | --- | --- | --- | --- |", + ] + iov = payload["invalid_over_valid_per_arm"] + for arm in payload["arms"]: + s = payload["arm_summaries"][arm] + lines.append( + f"| {arm} | {s['n']} | {f(s['valid_rate'])} | " + f"{f(s['mean_hard_rank'])} | {iov.get(arm, '—')} |" + ) + lines += [ + "", + "## Paired outcomes vs `ar_only` (per-example, nothing aggregated away)", + "", + "| arm | both valid | AR valid, arm invalid | AR invalid, arm valid | both invalid | unpaired |", + "| --- | --- | --- | --- | --- | --- |", + ] + for arm, counts in payload["paired_vs_ar_only"].items(): + lines.append( + f"| {arm} | {counts['both_valid']} | {counts['a_valid_b_invalid']} | " + f"{counts['a_invalid_b_valid']} | {counts['both_invalid']} | {counts['unpaired']} |" + ) + gates = payload["gates"] + wilson = gates["value"]["wilson"] + lines += [ + "", + "## Advancement gates", + "", + f"- **Safety**: invalid-over-valid = {gates['safety']['invalid_over_valid']} " + f"(must be 0) → {'PASS' if gates['safety']['pass'] else 'FAIL'}", + f"- **Value**: improvements {gates['value']['improvements']}/{gates['value']['n_paired']}, " + f"Wilson 95% [{wilson['low']}, {wilson['high']}] vs minimum useful effect " + f"{gates['value']['min_useful_effect']} → {'PASS' if gates['value']['pass'] else 'FAIL'}", + f"- **Reachability/provenance**: slm299/291 artifacts present → " + f"{'PASS' if gates['reachability']['pass'] else 'FAIL'}", + "", + "## Commit reasons", + "", + "| arm | reason | count |", + "| --- | --- | --- |", + ] + for arm, reasons in payload["commit_reason_counts"].items(): + for reason, count in reasons.items(): + lines.append(f"| {arm} | {reason} | {count} |") + lines += [ + "", + f"Oracle commit upper bound hard-valid rate: " + f"{f(payload['oracle_upper_bound']['valid_rate'])} " + f"(sanity ≥ improved hybrid: {payload['oracle_upper_bound']['sanity']}).", + "", + f"Durable per-example commit decisions: `{payload['decisions_path']}`.", + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=6) + parser.add_argument("--lr", type=float, default=3e-3) + parser.add_argument("--n-train", type=int, default=16) + parser.add_argument("--n-eval", type=int, default=8) + parser.add_argument("--seeds", type=int, nargs="+", default=[0, 1]) + parser.add_argument("--scorer-steps", type=int, default=20) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + parser.add_argument("--decisions-out", type=Path, default=DEFAULT_DECISIONS_OUT) + args = parser.parse_args(argv) + + payload = run_screen( + n_train=args.n_train, + n_eval=args.n_eval, + seeds=tuple(args.seeds), + steps=args.steps, + lr=args.lr, + scorer_steps=args.scorer_steps, + decisions_out=args.decisions_out, + ) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"disposition={payload['disposition']} " + f"safety={payload['gates']['safety']['pass']} " + f"value={payload['gates']['value']['pass']} " + f"iov={payload['invalid_over_valid_per_arm'].get('ar_repair_improved')} " + f"-> {args.json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm317_repair_hybrid.py b/src/slm_training/harnesses/experiments/slm317_repair_hybrid.py new file mode 100644 index 000000000..2634f176d --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm317_repair_hybrid.py @@ -0,0 +1,446 @@ +"""SLM-317 (LAR2-06): do-no-harm AR→repair hybrid commit machinery. + +A valid-state repair arm is only useful if it never damages a correct AR +program. This module provides the deterministic, metamorphism-invariant +commit rule for the AR→repair hybrid screen: + +- :func:`hard_evidence` scores one final-program source on the LAR1-02 + unified ladder: official parse (``dsl.parser.validate``) → output contract + (``output_contract_violations``) → ``meaningful_program_v1`` verdict. The + ladder rank is the ONLY hard authority. +- :func:`commit` applies the preregistered do-no-harm rule: + + a. hard evidence improves → COMMIT (``hard_improvement``); + b. hard evidence regresses → RETAIN source (``hard_regression``); + c. hard evidence unchanged and a calibrated learned score (model value or + oracle distance improvement) improves → COMMIT + (``soft_improvement_no_hard_regression``); + d. otherwise RETAIN (``no_improvement``); a missing/empty candidate is an + ABSTAIN (``candidate_unavailable``), never a silent commit. + +- :func:`oracle_commit` is the non-promotable upper bound: commit exactly + when the candidate's hard evidence is strictly better. +- Metamorphic transforms (:func:`alpha_rename`, :func:`reorder_statements`, + :func:`normalize_formatting`, :func:`ast_roundtrip`) are + semantics-preserving source rewrites used by the invariance tests: hard + evidence and commit decisions must be identical across them. +- :func:`repair_decode` runs the branch's value-guided beam repair starting + from an arbitrary valid source state (not the minimal seed), fail-closed + via ``TreeEditSpace.apply``. + +The module is torch-free at import time; ``repair_decode``/``state_value`` +import torch lazily through the model they are given. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any + +from slm_training.dsl.language_contract import output_contract_violations +from slm_training.dsl.parser import validate +from slm_training.harnesses.model_build.eval_runner import ( + meaningful_program_v1_report, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_STOP, + Statement, + parse_statements, + render_statements, +) + +EXPERIMENT_ID = "slm317-repair-hybrid" +COMMIT_DECISION_SCHEMA = "slm317_commit_decision/v1" + +# Commit actions. +ACTION_COMMIT = "commit" +ACTION_RETAIN = "retain" +ACTION_ABSTAIN = "abstain" + +# Preregistered reason codes (the full closed set). +REASON_HARD_IMPROVEMENT = "hard_improvement" +REASON_HARD_REGRESSION = "hard_regression" +REASON_SOFT_IMPROVEMENT = "soft_improvement_no_hard_regression" +REASON_NO_IMPROVEMENT = "no_improvement" +REASON_CANDIDATE_UNAVAILABLE = "candidate_unavailable" +REASON_ORACLE_HARD_IMPROVEMENT = "oracle_hard_improvement" +REASON_ORACLE_NO_GAIN = "oracle_no_gain" + +# Dispositions (exactly one per screen run). +DISPOSITION_POSITIVE = "repair_positive" +DISPOSITION_NEGATIVE = "repair_negative" +DISPOSITION_INCONCLUSIVE = "inconclusive" + + +# --------------------------------------------------------------------------- # +# Hard evidence (LAR1-02 ladder) +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class HardEvidence: + """Deterministic verifier/contract evidence for one final program.""" + + parse_ok: bool + contract_ok: bool + v1_verdict: bool + v1_reason_codes: tuple[str, ...] + contract_violations: tuple[str, ...] + + @property + def rank(self) -> int: + """Ordered ladder: 0 unparseable < 1 parsed < 2 contract-clean < 3 v1-valid.""" + if not self.parse_ok: + return 0 + if not self.contract_ok: + return 1 + if not self.v1_verdict: + return 2 + return 3 + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["v1_reason_codes"] = list(self.v1_reason_codes) + data["contract_violations"] = list(self.contract_violations) + data["rank"] = self.rank + return data + + +def hard_evidence(source: str | None) -> HardEvidence: + """Score one program source on the deterministic ladder. Never raises.""" + if not source: + return HardEvidence(False, False, False, ("no_program",), ()) + parse_ok = False + try: + validate(source) + parse_ok = True + except Exception: # noqa: BLE001 - parse fact, never raised + parse_ok = False + violations: tuple[str, ...] = () + if parse_ok: + violations = tuple(output_contract_violations(source)) + contract_ok = parse_ok and not violations + report = meaningful_program_v1_report(source, gold=None) + return HardEvidence( + parse_ok=parse_ok, + contract_ok=contract_ok, + v1_verdict=bool(report["verdict"]), + v1_reason_codes=tuple(report.get("reason_codes", ())), + contract_violations=violations, + ) + + +# --------------------------------------------------------------------------- # +# Commit rule +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class CommitDecision: + """Durable per-example commit record (source + candidate + exact reason).""" + + action: str # commit | retain | abstain + reason: str + source: str + candidate: str + final: str + source_evidence: HardEvidence + candidate_evidence: HardEvidence + learned_score_source: float | None = None + learned_score_candidate: float | None = None + record_id: str = "" + arm_id: str = "" + seed: int = 0 + notes: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": COMMIT_DECISION_SCHEMA, + "action": self.action, + "reason": self.reason, + "source": self.source, + "candidate": self.candidate, + "final": self.final, + "source_evidence": self.source_evidence.to_dict(), + "candidate_evidence": self.candidate_evidence.to_dict(), + "learned_score_source": self.learned_score_source, + "learned_score_candidate": self.learned_score_candidate, + "record_id": self.record_id, + "arm_id": self.arm_id, + "seed": self.seed, + "notes": list(self.notes), + } + + +def commit( + source: str, + candidate: str | None, + *, + evidence: dict[str, Any] | None = None, + record_id: str = "", + arm_id: str = "", + seed: int = 0, +) -> CommitDecision: + """Do-no-harm commit rule (deterministic; see module docstring). + + ``evidence`` may carry ``learned_score_source`` / + ``learned_score_candidate`` (model value or oracle distance improvement; + higher = better) and ``learned_margin`` (strict improvement margin, + default 0.0). Learned scores are only consulted when hard evidence is + unchanged, and a soft commit is forbidden whenever hard evidence regresses. + """ + evidence = evidence or {} + src_ev = hard_evidence(source) + if candidate is None or not candidate.strip(): + return CommitDecision( + action=ACTION_ABSTAIN, + reason=REASON_CANDIDATE_UNAVAILABLE, + source=source, + candidate="", + final=source, + source_evidence=src_ev, + candidate_evidence=hard_evidence(None), + record_id=record_id, + arm_id=arm_id, + seed=seed, + ) + cand_ev = hard_evidence(candidate) + soft_s = evidence.get("learned_score_source") + soft_c = evidence.get("learned_score_candidate") + margin = float(evidence.get("learned_margin", 0.0)) + + if cand_ev.rank > src_ev.rank: + action, reason, final = ACTION_COMMIT, REASON_HARD_IMPROVEMENT, candidate + elif cand_ev.rank < src_ev.rank: + action, reason, final = ACTION_RETAIN, REASON_HARD_REGRESSION, source + elif ( + soft_s is not None + and soft_c is not None + and float(soft_c) > float(soft_s) + margin + ): + action, reason, final = ACTION_COMMIT, REASON_SOFT_IMPROVEMENT, candidate + else: + action, reason, final = ACTION_RETAIN, REASON_NO_IMPROVEMENT, source + return CommitDecision( + action=action, + reason=reason, + source=source, + candidate=candidate, + final=final, + source_evidence=src_ev, + candidate_evidence=cand_ev, + learned_score_source=None if soft_s is None else float(soft_s), + learned_score_candidate=None if soft_c is None else float(soft_c), + record_id=record_id, + arm_id=arm_id, + seed=seed, + ) + + +def oracle_commit( + source: str, + candidate: str | None, + *, + record_id: str = "", + arm_id: str = "", + seed: int = 0, +) -> CommitDecision: + """Non-promotable upper bound: commit exactly on strict hard improvement.""" + if candidate is None or not candidate.strip(): + return commit( + source, candidate, record_id=record_id, arm_id=arm_id, seed=seed + ) + src_ev = hard_evidence(source) + cand_ev = hard_evidence(candidate) + better = cand_ev.rank > src_ev.rank + return CommitDecision( + action=ACTION_COMMIT if better else ACTION_RETAIN, + reason=REASON_ORACLE_HARD_IMPROVEMENT if better else REASON_ORACLE_NO_GAIN, + source=source, + candidate=candidate, + final=candidate if better else source, + source_evidence=src_ev, + candidate_evidence=cand_ev, + record_id=record_id, + arm_id=arm_id, + seed=seed, + notes=["oracle selector; non-promotable upper bound"], + ) + + +# --------------------------------------------------------------------------- # +# Metamorphic transforms (semantics-preserving; used by invariance tests) +# --------------------------------------------------------------------------- # + + +def alpha_rename(source: str, *, prefix: str = "m") -> str: + """Consistently rename non-root node identifiers (references included).""" + statements = parse_statements(source) + if statements is None: + return source + mapping: dict[str, str] = {} + for stmt in statements: + if stmt.name != "root": + mapping[stmt.name] = f"{prefix}{len(mapping)}" + renamed = [ + Statement( + name=mapping.get(stmt.name, stmt.name), + comp=stmt.comp, + children=[mapping.get(c, c) for c in stmt.children], + rest=stmt.rest, + has_list=stmt.has_list, + ) + for stmt in statements + ] + return render_statements(renamed) + + +def reorder_statements(source: str) -> str: + """Move the root statement last (declarations first); semantics preserved.""" + statements = parse_statements(source) + if statements is None: + return source + roots = [s for s in statements if s.name == "root"] + others = [s for s in statements if s.name != "root"] + return render_statements(others + roots) + + +def normalize_formatting(source: str) -> str: + """Re-render the structural form (canonical spacing/newlines).""" + statements = parse_statements(source) + if statements is None: + return source.strip() + return render_statements(statements) + + +def ast_roundtrip(source: str) -> str: + """Equivalent AST serialization via the official parser's serialized form.""" + try: + program = validate(source) + except Exception: # noqa: BLE001 - unparseable input round-trips to itself + return source + return program.serialized or source.strip() + + +# --------------------------------------------------------------------------- # +# Repair decode from an arbitrary valid source state +# --------------------------------------------------------------------------- # + + +def repair_decode( + model: Any, + source: str, + inventory: list[str], + prompt: str, +) -> tuple[str, dict[str, Any]]: + """Value-guided beam repair seeded from ``source`` (not the minimal seed). + + Mirrors ``TreeEditDiffusionModel._decode_one`` (same fail-closed + ``TreeEditSpace.apply``, same value re-scoring, same STOP handling) with + the beam initialized on the caller's valid state. An unparseable or + invalid source fails closed: returns ``("", {"failure": ...})``. + """ + import torch # lazy: module import stays torch-free + + statements = parse_statements(source) + if statements is None: + return "", {"failure": "source_unparseable", "kind": "tree_edit_repair"} + from slm_training.models.tree_edit_diffusion import _is_valid + + if not _is_valid(source): + return "", {"failure": "source_invalid", "kind": "tree_edit_repair"} + + model.eval() + ctx, ctx_pad = model._encode_context( + [model._format_context(prompt, slot_contract=list(inventory))] + ) + cfg = model.config + beam: list[tuple[float, list[Statement], bool]] = [(0.0, statements, False)] + evidence: dict[str, Any] = {"steps": 0, "expansions": 0, "kind": "tree_edit_repair"} + with torch.no_grad(): + for _ in range(cfg.max_search_steps): + live = [entry for entry in beam if not entry[2]] + if not live: + break + sources = [render_statements(s) for _, s, _ in live] + out = model.policy( + model._state_batch(sources), + model.tokenizer.pad_id, + ctx.expand(len(sources), -1, -1), + ctx_pad.expand(len(sources), -1), + ) + next_beam = [entry for entry in beam if entry[2]] + seen: set[str] = { + render_statements(s) for _, s, frozen in next_beam if frozen + } + for row, (_, stmts, _) in enumerate(live): + candidates = model._enumerate_edits(out, row, len(stmts), len(inventory)) + expanded = 0 + for score, edit in candidates: + if expanded >= cfg.expand_per_state: + break + if edit.action == ACTION_STOP: + text = render_statements(stmts) + if text in seen: + continue + seen.add(text) + next_beam.append((float(out["value"][row]), stmts, True)) + expanded += 1 + continue + child = model.space.apply(stmts, edit, inventory, reason=[]) + if child is None: + continue + text = render_statements(child) + if text in seen: + continue + seen.add(text) + next_beam.append((float(out["value"][row]), child, False)) + expanded += 1 + evidence["expansions"] += 1 + if not next_beam: + break + unfrozen = [entry for entry in next_beam if not entry[2]] + if unfrozen: + rescore = model.policy( + model._state_batch([render_statements(s) for _, s, _ in unfrozen]), + model.tokenizer.pad_id, + ctx.expand(len(unfrozen), -1, -1), + ctx_pad.expand(len(unfrozen), -1), + ) + rescored = [ + (float(rescore["value"][i]), entry[1], False) + for i, entry in enumerate(unfrozen) + ] + else: + rescored = [] + frozen = [entry for entry in next_beam if entry[2]] + beam = sorted(frozen + rescored, key=lambda e: e[0], reverse=True)[ + : cfg.beam_width + ] + evidence["steps"] += 1 + if all(entry[2] for entry in beam): + break + best = max(beam, key=lambda e: e[0]) + evidence["value"] = float(best[0]) + evidence["frozen"] = bool(best[2]) + return render_statements(best[1]), evidence + + +def state_value(model: Any, source: str, inventory: list[str], prompt: str) -> float | None: + """Calibrated learned score (policy value head) for one state; None if unparseable.""" + import torch # lazy + + if parse_statements(source) is None: + return None + model.eval() + ctx, ctx_pad = model._encode_context( + [model._format_context(prompt, slot_contract=list(inventory))] + ) + with torch.no_grad(): + out = model.policy( + model._state_batch([source]), + model.tokenizer.pad_id, + ctx, + ctx_pad, + ) + return float(out["value"][0]) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 0edc43962..dadbf251a 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -7879,6 +7879,24 @@ "note": "initial registration; SLM-314 LAR2-05 winner-take-all (MCL): frozen 8-prompt x 2-mode verifier-accepted canonical-AST dataset (alpha-invariant fingerprints + sha256 manifest), min-loss-mode loss with preregistered 0.1 coverage/floor term, deterministic two-mode synthetic collapse-vs-retain proof, matched {single_gold, multi_gold, wta} arms" } ] + }, + "harness.experiments.slm317_repair_hybrid": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm317_repair_hybrid.py", + "scripts/run_slm317_repair_hybrid.py", + "tests/test_harnesses/experiments/test_slm317_repair_hybrid.py", + "docs/design/iter-slm317-repair-hybrid-20260724.json", + "docs/design/iter-slm317-repair-hybrid-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-317 LAR2-06 do-no-harm AR->repair hybrid: metamorphism-invariant commit rule (hard ladder + calibrated soft score), matched arms incl. historical/improved repair + oracle upper bound, invalid-over-valid visibility, advancement gates and repair_positive|repair_negative|inconclusive disposition" + } + ] } } } diff --git a/tests/test_harnesses/experiments/test_slm317_repair_hybrid.py b/tests/test_harnesses/experiments/test_slm317_repair_hybrid.py new file mode 100644 index 000000000..76c7a93bf --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm317_repair_hybrid.py @@ -0,0 +1,263 @@ +"""SLM-317 (LAR2-06): metamorphic invariance, commit truth table, visibility. + +Runs before any model evaluation: the commit/score pipeline must be invariant +to alpha-renaming, semantics-preserving reordering, formatting normalization, +and equivalent AST serializations — and the do-no-harm rule must cover its +full truth table deterministically. +""" + +from __future__ import annotations + +import json + +import pytest + +from slm_training.harnesses.experiments.slm155_factorization_comparison import ( + paired_validity_table, +) +from slm_training.harnesses.experiments.slm317_repair_hybrid import ( + ACTION_ABSTAIN, + ACTION_COMMIT, + ACTION_RETAIN, + REASON_CANDIDATE_UNAVAILABLE, + REASON_HARD_IMPROVEMENT, + REASON_HARD_REGRESSION, + REASON_NO_IMPROVEMENT, + REASON_ORACLE_NO_GAIN, + REASON_SOFT_IMPROVEMENT, + alpha_rename, + ast_roundtrip, + commit, + hard_evidence, + normalize_formatting, + oracle_commit, + reorder_statements, +) +from scripts.run_slm317_repair_hybrid import ArmOutcome + +VALID = 'root = Stack([n0], "column")\nn0 = TextContent(":content")' +VALID_TWO = ( + 'root = Stack([n0, n1], "column")\n' + 'n0 = TextContent(":content")\n' + 'n1 = Button(":cta")' +) +INVALID_UNPARSEABLE = "root = Stack(([n0]," +VALID_PARSE_ONLY = 'root = Stack([], "column")' # parses; v1 invalid (empty stack) + +TRANSFORMS = (alpha_rename, reorder_statements, normalize_formatting, ast_roundtrip) + + +# --------------------------------------------------------------------------- # +# Metamorphic invariance (all four transforms) +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("source", [VALID, VALID_TWO, VALID_PARSE_ONLY]) +def test_hard_evidence_invariant_across_transforms(source: str) -> None: + base = hard_evidence(source) + for transform in TRANSFORMS: + variant = transform(source) + ev = hard_evidence(variant) + assert ev.rank == base.rank, transform.__name__ + assert ev.parse_ok == base.parse_ok + assert ev.contract_ok == base.contract_ok + assert ev.v1_verdict == base.v1_verdict + + +def test_commit_decision_invariant_across_transforms() -> None: + evidence = {"learned_score_source": 0.2, "learned_score_candidate": 0.9} + base = commit(VALID, VALID_TWO, evidence=evidence) + for transform in TRANSFORMS: + variant = commit(transform(VALID), transform(VALID_TWO), evidence=evidence) + assert variant.action == base.action, transform.__name__ + assert variant.reason == base.reason, transform.__name__ + assert variant.source_evidence.rank == base.source_evidence.rank + assert variant.candidate_evidence.rank == base.candidate_evidence.rank + + +def test_transforms_preserve_validity() -> None: + for transform in TRANSFORMS: + assert hard_evidence(transform(VALID)).v1_verdict, transform.__name__ + + +# --------------------------------------------------------------------------- # +# Commit-rule truth table (all branches + determinism) +# --------------------------------------------------------------------------- # + + +def test_truth_table_hard_improvement() -> None: + decision = commit(VALID_PARSE_ONLY, VALID) + assert decision.action == ACTION_COMMIT + assert decision.reason == REASON_HARD_IMPROVEMENT + assert decision.final == VALID + + +def test_truth_table_hard_regression_retains_source() -> None: + decision = commit(VALID, INVALID_UNPARSEABLE) + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_HARD_REGRESSION + assert decision.final == VALID + + +def test_truth_table_soft_improvement_no_hard_regression() -> None: + candidate = alpha_rename(VALID) # same hard rank, different source + decision = commit( + VALID, + candidate, + evidence={"learned_score_source": 0.1, "learned_score_candidate": 0.9}, + ) + assert decision.action == ACTION_COMMIT + assert decision.reason == REASON_SOFT_IMPROVEMENT + assert decision.final == candidate + + +def test_soft_commit_forbidden_when_hard_regresses() -> None: + decision = commit( + VALID, + INVALID_UNPARSEABLE, + evidence={"learned_score_source": 0.1, "learned_score_candidate": 0.9}, + ) + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_HARD_REGRESSION + + +def test_truth_table_no_improvement() -> None: + candidate = alpha_rename(VALID) + decision = commit(VALID, candidate) # no learned scores, same rank + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_NO_IMPROVEMENT + assert decision.final == VALID + + +def test_soft_margin_not_exceeded_retains() -> None: + candidate = alpha_rename(VALID) + decision = commit( + VALID, + candidate, + evidence={ + "learned_score_source": 0.5, + "learned_score_candidate": 0.5, + "learned_margin": 0.0, + }, + ) + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_NO_IMPROVEMENT + + +def test_truth_table_abstain_on_missing_candidate() -> None: + for candidate in (None, "", " "): + decision = commit(VALID, candidate) + assert decision.action == ACTION_ABSTAIN + assert decision.reason == REASON_CANDIDATE_UNAVAILABLE + assert decision.final == VALID + + +def test_commit_deterministic() -> None: + evidence = {"learned_score_source": 0.1, "learned_score_candidate": 0.9} + first = commit(VALID, VALID_TWO, evidence=evidence).to_dict() + second = commit(VALID, VALID_TWO, evidence=evidence).to_dict() + assert first == second + + +def test_decision_preserves_source_candidate_reason() -> None: + decision = commit(VALID_PARSE_ONLY, VALID, record_id="r1", arm_id="a", seed=3) + row = decision.to_dict() + assert row["source"] == VALID_PARSE_ONLY + assert row["candidate"] == VALID + assert row["reason"] == REASON_HARD_IMPROVEMENT + assert row["record_id"] == "r1" + assert row["schema"] == "slm317_commit_decision/v1" + json.dumps(row) # durable JSONL-serializable + + +# --------------------------------------------------------------------------- # +# Oracle selector upper bound +# --------------------------------------------------------------------------- # + + +def test_oracle_commits_only_on_strict_hard_improvement() -> None: + better = oracle_commit(VALID_PARSE_ONLY, VALID) + assert better.action == ACTION_COMMIT + worse = oracle_commit(VALID, VALID_PARSE_ONLY) + assert worse.action == ACTION_RETAIN + assert worse.reason == REASON_ORACLE_NO_GAIN + equal = oracle_commit(VALID, alpha_rename(VALID)) + assert equal.action == ACTION_RETAIN # equal rank is not an improvement + + +def test_oracle_upper_bound_dominates_rule() -> None: + # Whatever the deterministic rule commits, the oracle commits a superset + # (it commits on every strict hard improvement; the rule adds soft commits + # only when hard evidence is unchanged, which can never beat the oracle's + # final rank). + cases = [ + (VALID_PARSE_ONLY, VALID), + (VALID, VALID_PARSE_ONLY), + (VALID, INVALID_UNPARSEABLE), + (VALID, alpha_rename(VALID)), + ] + for source, candidate in cases: + rule = commit( + source, + candidate, + evidence={"learned_score_source": 0.0, "learned_score_candidate": 1.0}, + ) + oracle = oracle_commit(source, candidate) + assert oracle.source_evidence.rank <= oracle.candidate_evidence.rank or ( + oracle.action == ACTION_RETAIN + ) + assert hard_evidence(oracle.final).rank >= hard_evidence(rule.final).rank + + +# --------------------------------------------------------------------------- # +# Invalid-over-valid visibility + arm isolation +# --------------------------------------------------------------------------- # + + +def _outcome(record_id: str, arm: str, seed: int, verdict: bool) -> ArmOutcome: + return ArmOutcome( + record_id=record_id, + arm_id=arm, + seed=seed, + final_source=VALID if verdict else INVALID_UNPARSEABLE, + v1_verdict=verdict, + hard_rank=3 if verdict else 0, + ) + + +def test_invalid_over_valid_damage_surfaces() -> None: + """AR final valid + repaired final invalid must be visible, never hidden.""" + ar = [_outcome("r0", "ar_only", 0, True), _outcome("r1", "ar_only", 0, True)] + repaired = [ + _outcome("r0", "ar_repair_improved", 0, True), + _outcome("r1", "ar_repair_improved", 0, False), # damaged + ] + counts = paired_validity_table(ar, repaired) + assert counts["a_valid_b_invalid"] == 1 # the damage count is explicit + assert counts["both_valid"] == 1 + + +def test_commit_rule_makes_damage_unreachable() -> None: + """With the commit rule in the loop, a damaging candidate is retained away.""" + decision = commit(VALID, INVALID_UNPARSEABLE) + final_ev = hard_evidence(decision.final) + assert final_ev.v1_verdict # final is the retained valid AR source + + +def test_arm_isolation() -> None: + """Outcomes from one arm never leak into another arm's pairing.""" + ar = [_outcome("r0", "ar_only", 0, True)] + other = [_outcome("r0", "repair_only", 0, False)] + counts = paired_validity_table(ar, other) + assert counts["a_valid_b_invalid"] == 1 + # A third arm with no matching record contributes only unpaired rows. + third = [_outcome("rX", "ar_repair_improved", 0, True)] + counts2 = paired_validity_table(ar, third) + assert counts2["unpaired"] == 2 + assert counts2["a_valid_b_invalid"] == 0 + + +def test_paired_table_deterministic() -> None: + ar = [_outcome(f"r{i}", "ar_only", 0, i % 2 == 0) for i in range(4)] + arm = [_outcome(f"r{i}", "ar_repair_improved", 0, True) for i in range(4)] + assert paired_validity_table(ar, arm) == paired_validity_table(ar, arm) From e258c9173c2da0637ae41ed2c4405183f9e610b2 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:28:28 -0500 Subject: [PATCH 10/18] =?UTF-8?q?SLM-319:=20close=20LAR3-01=20not=5Fauthor?= =?UTF-8?q?ized=20=E2=80=94=20entry=20gates=20unmet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LAR0-02 returned recursive_core_negative (1 of 2 required seed passes) and LAR2-06 returned inconclusive with the value gate failed (0/16, Wilson [0.0, 0.194] vs 0.05); LAR3 is open but NOT advanced. No production code, defaults, or checkpoints; LAR3-02..05 stay blocked. Reopening requires recursive_core_positive plus a passing repair advancement screen. --- ...er-slm319-lar3-gate-closeout-20260725.json | 49 +++++++++++++++++++ ...iter-slm319-lar3-gate-closeout-20260725.md | 33 +++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 docs/design/iter-slm319-lar3-gate-closeout-20260725.json create mode 100644 docs/design/iter-slm319-lar3-gate-closeout-20260725.md diff --git a/docs/design/iter-slm319-lar3-gate-closeout-20260725.json b/docs/design/iter-slm319-lar3-gate-closeout-20260725.json new file mode 100644 index 000000000..8dfa80de0 --- /dev/null +++ b/docs/design/iter-slm319-lar3-gate-closeout-20260725.json @@ -0,0 +1,49 @@ +{ + "claim_class": "wiring", + "decision": "not_authorized", + "downstream_effect": "SLM-321/324/326/327 (LAR3-02..05) stay blocked on SLM-319. Reopen only when a recurrence-health audit returns recursive_core_positive AND a valid-state repair advancement screen passes its value gate.", + "failed_gates": [ + "gate_lar0_02", + "gate_lar2_06" + ], + "gates": { + "gate_lar0_01": { + "issue": "SLM-279", + "observed_outcome": "Done (correction-only); neutral for entry", + "passed": true, + "required_outcome": "canonical depth-supervision objective defined" + }, + "gate_lar0_02": { + "evidence": "docs/design/iter-slm282-recurrence-health-20260723.json", + "issue": "SLM-282", + "observed_outcome": "recursive_core_negative (1 of 2 required seed passes)", + "passed": false, + "required_outcome": "recursive_core_positive (>=2 seed passes)" + }, + "gate_lar2_06": { + "evidence": "docs/design/iter-slm317-repair-hybrid-20260724.json", + "issue": "SLM-317", + "observed_outcome": "inconclusive; safety PASS, value FAIL (0/16, Wilson [0.0, 0.194] vs 0.05); LAR3 open but NOT advanced", + "passed": false, + "required_outcome": "advancement screen passes safety + value gates" + } + }, + "issue": "SLM-319", + "matrix_set": "slm319-prelude-coda-core", + "matrix_version": "slm319-v1", + "production_code_added": false, + "production_default_changed": false, + "reason": "LAR3 entry prerequisites are not met: SLM-282 (LAR0-02) returned recursive_core_negative (1 of 2 required seed passes on the preregistered recurrence-health condition), and SLM-317 (LAR2-06) returned inconclusive with LAR3 explicitly 'open but NOT advanced' after the value gate failed (0/16 improvements, Wilson [0.0, 0.194] vs 0.05). Per the program's own advancement rules, the untied prelude/shared-core/coda architecture must not be implemented.", + "run_id": "slm319_gate_closeout", + "schema": "slm319_lar3_gate_closeout/v1", + "status": "closeout", + "version_stamp": { + "code_commit": "48e5cadc13f9e6e431491da615badb4a0614c17a", + "code_dirty": false, + "components": { + "harness.experiments": "v83" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:27:51.049596+00:00" + } +} diff --git a/docs/design/iter-slm319-lar3-gate-closeout-20260725.md b/docs/design/iter-slm319-lar3-gate-closeout-20260725.md new file mode 100644 index 000000000..c93ff7983 --- /dev/null +++ b/docs/design/iter-slm319-lar3-gate-closeout-20260725.md @@ -0,0 +1,33 @@ +# SLM-319 / LAR3-01: LAR3 entry-gate closeout (slm319_gate_closeout) + +Matrix set: `slm319-prelude-coda-core` · Version: `slm319-v1` · Status: **closeout** +Decision: **not_authorized** — no production code. + +## Gate assessment + +The issue is blocked by LAR0-01, LAR0-02, and LAR2-06; its own advancement +rules ("Only `recursive_core_positive` can satisfy a prerequisite for LAR3"; +"Compare only after LAR3 entry gates pass") require all of them. They do not +pass. + +| Gate | Issue | Required | Observed | Passed | +| --- | --- | --- | --- | --- | +| LAR0-01 | SLM-279 | canonical depth-supervision objective | Done (correction-only); neutral | True | +| LAR0-02 | SLM-282 | `recursive_core_positive` (≥2 seed passes) | `recursive_core_negative` — 1 of 2 required seed passes ([json](iter-slm282-recurrence-health-20260723.json)) | **False** | +| LAR2-06 | SLM-317 | advancement screen passes safety + value gates | `inconclusive` — safety PASS, value FAIL (0/16, Wilson [0.0, 0.194] vs 0.05); LAR3 open but NOT advanced ([json](iter-slm317-repair-hybrid-20260724.json)) | **False** | + +## Consequences + +- No prelude/shared-core/coda architecture is implemented; no config, + checkpoint, or default change. +- SLM-321/324/326/327 (LAR3-02..05) stay blocked on this issue. + +## Reopening conditions + +Reopen or supersede only when **both**: + +1. a recurrence-health audit returns `recursive_core_positive` (≥2 seeds on + the preregistered condition), and +2. a valid-state repair advancement screen passes its preregistered value + gate (paired semantic improvement with Wilson lower bound above the + minimum useful effect). From a783514e4f6419528738f8151d1e6f3979375eda Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:31:25 -0500 Subject: [PATCH 11/18] =?UTF-8?q?SLM-321:=20close=20LAR3-02=20not=5Fauthor?= =?UTF-8?q?ized=20=E2=80=94=20LAR3-01=20closed=20and=20delta=20premise=20n?= =?UTF-8?q?egative?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SLM-319 closed not_authorized (LAR3 entry gates unmet), so the recurrent-core abstraction this issue anchors on does not exist; independently SLM-282's matched audit found the as_is arm negative (1/2 seeds) and residual_delta_can_promote=false. No production code. --- ...er-slm321-lar3-gate-closeout-20260725.json | 43 +++++++++++++++++++ ...iter-slm321-lar3-gate-closeout-20260725.md | 21 +++++++++ 2 files changed, 64 insertions(+) create mode 100644 docs/design/iter-slm321-lar3-gate-closeout-20260725.json create mode 100644 docs/design/iter-slm321-lar3-gate-closeout-20260725.md diff --git a/docs/design/iter-slm321-lar3-gate-closeout-20260725.json b/docs/design/iter-slm321-lar3-gate-closeout-20260725.json new file mode 100644 index 000000000..26cd52610 --- /dev/null +++ b/docs/design/iter-slm321-lar3-gate-closeout-20260725.json @@ -0,0 +1,43 @@ +{ + "claim_class": "wiring", + "decision": "not_authorized", + "downstream_effect": "SLM-324/326/327 stay blocked. Reopen with SLM-319 when a recurrence-health audit returns recursive_core_positive AND a repair advancement screen passes its value gate.", + "failed_gates": [ + "gate_lar3_01", + "gate_recurrence_health" + ], + "gates": { + "gate_lar3_01": { + "evidence": "docs/design/iter-slm319-lar3-gate-closeout-20260725.json", + "issue": "SLM-319", + "observed_outcome": "closed not_authorized (entry gates unmet)", + "passed": false, + "required_outcome": "LAR3-01 recurrent-core abstraction implemented" + }, + "gate_recurrence_health": { + "evidence": "docs/design/iter-slm282-recurrence-health-20260723.json", + "issue": "SLM-282", + "observed_outcome": "recursive_core_negative (1/2 seeds); residual_delta_can_promote=false", + "passed": false, + "required_outcome": "recursive_core_positive or promotable residual_delta arm" + } + }, + "issue": "SLM-321", + "matrix_set": "slm321-residual-delta-gates", + "matrix_version": "slm321-v1", + "production_code_added": false, + "production_default_changed": false, + "reason": "SLM-319 (LAR3-01) closed not_authorized on 2026-07-25: LAR3 entry prerequisites fail (SLM-282 recursive_core_negative, SLM-317 inconclusive with the value gate failed), so the LAR3-01 recurrent-core abstraction this issue anchors on does not exist. Independently, SLM-282's matched as_is/residual_delta audit found the as_is arm negative (1 of 2 required seed passes) and residual_delta_can_promote=false, so the delta-mode premise has no positive fixture evidence either.", + "run_id": "slm321_gate_closeout", + "schema": "slm321_lar3_gate_closeout/v1", + "status": "closeout", + "version_stamp": { + "code_commit": "e258c9173c2da0637ae41ed2c4405183f9e610b2", + "code_dirty": false, + "components": { + "harness.experiments": "v83" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:30:46.372520+00:00" + } +} diff --git a/docs/design/iter-slm321-lar3-gate-closeout-20260725.md b/docs/design/iter-slm321-lar3-gate-closeout-20260725.md new file mode 100644 index 000000000..cd016ec4c --- /dev/null +++ b/docs/design/iter-slm321-lar3-gate-closeout-20260725.md @@ -0,0 +1,21 @@ +# SLM-321 / LAR3-02: LAR3 gate closeout (slm321_gate_closeout) + +Matrix set: `slm321-residual-delta-gates` · Version: `slm321-v1` · Status: **closeout** +Decision: **not_authorized** — no production code. + +## Gate assessment + +| Gate | Issue | Required | Observed | Passed | +| --- | --- | --- | --- | --- | +| LAR3-01 core abstraction | SLM-319 | recurrent-core abstraction implemented | closed `not_authorized` — LAR3 entry gates unmet ([json](iter-slm319-lar3-gate-closeout-20260725.json)) | **False** | +| Recurrence health / delta premise | SLM-282 | `recursive_core_positive` or a promotable `residual_delta` arm | `recursive_core_negative` (1/2 required seeds); `residual_delta_can_promote=false` ([json](iter-slm282-recurrence-health-20260723.json)) | **False** | + +The residual-delta update mode this issue would advance already has matched +fixture evidence against it (SLM-282's audit), and the LAR3-01 abstraction it +anchors on was never built. No production code, defaults, or checkpoints. + +## Reopening conditions + +Reopen together with SLM-319 when a recurrence-health audit returns +`recursive_core_positive` AND a valid-state repair advancement screen passes +its preregistered value gate. From 480e59af94f3a1a732f2a539cbb1ae518fd540aa Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:32:47 -0500 Subject: [PATCH 12/18] =?UTF-8?q?SLM-324:=20close=20LAR3-03=20not=5Fauthor?= =?UTF-8?q?ized=20=E2=80=94=20LAR3-01/02=20both=20closed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...er-slm324-lar3-gate-closeout-20260725.json | 43 +++++++++++++++++++ ...iter-slm324-lar3-gate-closeout-20260725.md | 13 ++++++ 2 files changed, 56 insertions(+) create mode 100644 docs/design/iter-slm324-lar3-gate-closeout-20260725.json create mode 100644 docs/design/iter-slm324-lar3-gate-closeout-20260725.md diff --git a/docs/design/iter-slm324-lar3-gate-closeout-20260725.json b/docs/design/iter-slm324-lar3-gate-closeout-20260725.json new file mode 100644 index 000000000..26da726e6 --- /dev/null +++ b/docs/design/iter-slm324-lar3-gate-closeout-20260725.json @@ -0,0 +1,43 @@ +{ + "claim_class": "wiring", + "decision": "not_authorized", + "downstream_effect": "SLM-326/327 stay blocked. Reopens with SLM-319/321 when recursive_core_positive and a passing repair advancement screen exist.", + "failed_gates": [ + "gate_lar3_01", + "gate_lar3_02" + ], + "gates": { + "gate_lar3_01": { + "evidence": "docs/design/iter-slm319-lar3-gate-closeout-20260725.json", + "issue": "SLM-319", + "observed_outcome": "closed not_authorized (LAR3 entry gates unmet)", + "passed": false, + "required_outcome": "LAR3-01 shared semantic core implemented" + }, + "gate_lar3_02": { + "evidence": "docs/design/iter-slm321-lar3-gate-closeout-20260725.json", + "issue": "SLM-321", + "observed_outcome": "closed not_authorized (same gate evidence)", + "passed": false, + "required_outcome": "LAR3-02 update-mode disposition" + } + }, + "issue": "SLM-324", + "matrix_set": "slm324-loop-lora", + "matrix_version": "slm324-v1", + "production_code_added": false, + "production_default_changed": false, + "reason": "SLM-324 requires the LAR3-01 shared core (blocked by LAR3-01) and the LAR3-02 update-mode outcome (blocked by LAR3-02). SLM-319 closed not_authorized (LAR3 entry gates unmet) and SLM-321 closed not_authorized on the same evidence, so neither prerequisite exists. Loop-specific LoRA/FiLM specialization has no shared core to attach to.", + "run_id": "slm324_gate_closeout", + "schema": "slm324_lar3_gate_closeout/v1", + "status": "closeout", + "version_stamp": { + "code_commit": "a783514e4f6419528738f8151d1e6f3979375eda", + "code_dirty": false, + "components": { + "harness.experiments": "v83" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:32:47.534377+00:00" + } +} diff --git a/docs/design/iter-slm324-lar3-gate-closeout-20260725.md b/docs/design/iter-slm324-lar3-gate-closeout-20260725.md new file mode 100644 index 000000000..f66eca8c9 --- /dev/null +++ b/docs/design/iter-slm324-lar3-gate-closeout-20260725.md @@ -0,0 +1,13 @@ +# SLM-324 / LAR3-03: LAR3 gate closeout (slm324_gate_closeout) + +Matrix set: `slm324-loop-lora` · Version: `slm324-v1` · Status: **closeout** +Decision: **not_authorized** — no production code. + +| Gate | Issue | Required | Observed | Passed | +| --- | --- | --- | --- | --- | +| LAR3-01 shared core | SLM-319 | shared semantic core implemented | closed `not_authorized` ([json](iter-slm319-lar3-gate-closeout-20260725.json)) | **False** | +| LAR3-02 update modes | SLM-321 | update-mode disposition | closed `not_authorized` ([json](iter-slm321-lar3-gate-closeout-20260725.json)) | **False** | + +Loop-specific LoRA/FiLM specialization has no shared core to attach to. No +production code, defaults, or checkpoints. Reopens with SLM-319/321 when +`recursive_core_positive` and a passing repair advancement screen exist. From dc3b56ad170434ee2c5316e0384d89a5565a482d Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:34:15 -0500 Subject: [PATCH 13/18] =?UTF-8?q?SLM-326:=20close=20LAR3-04=20not=5Fauthor?= =?UTF-8?q?ized=20=E2=80=94=20LAR3-01..03=20all=20closed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...er-slm326-lar3-gate-closeout-20260725.json | 48 +++++++++++++++++++ ...iter-slm326-lar3-gate-closeout-20260725.md | 13 +++++ 2 files changed, 61 insertions(+) create mode 100644 docs/design/iter-slm326-lar3-gate-closeout-20260725.json create mode 100644 docs/design/iter-slm326-lar3-gate-closeout-20260725.md diff --git a/docs/design/iter-slm326-lar3-gate-closeout-20260725.json b/docs/design/iter-slm326-lar3-gate-closeout-20260725.json new file mode 100644 index 000000000..e02bb81fb --- /dev/null +++ b/docs/design/iter-slm326-lar3-gate-closeout-20260725.json @@ -0,0 +1,48 @@ +{ + "claim_class": "wiring", + "decision": "not_authorized", + "downstream_effect": "SLM-327 (LAR3-05) stays blocked; SLM-139 remains closed.", + "failed_gates": [ + "gate_lar3_01", + "gate_lar3_02", + "gate_lar3_03" + ], + "gates": { + "gate_lar3_01": { + "evidence": "docs/design/iter-slm319-lar3-gate-closeout-20260725.json", + "issue": "SLM-319", + "observed_outcome": "closed not_authorized", + "passed": false + }, + "gate_lar3_02": { + "evidence": "docs/design/iter-slm321-lar3-gate-closeout-20260725.json", + "issue": "SLM-321", + "observed_outcome": "closed not_authorized", + "passed": false + }, + "gate_lar3_03": { + "evidence": "docs/design/iter-slm324-lar3-gate-closeout-20260725.json", + "issue": "SLM-324", + "observed_outcome": "closed not_authorized", + "passed": false + } + }, + "issue": "SLM-326", + "matrix_set": "slm326-stochastic-loops", + "matrix_version": "slm326-v1", + "production_code_added": false, + "production_default_changed": false, + "reason": "SLM-326 requires LAR3-01 through LAR3-03. All three are closed not_authorized on the same LAR3 entry-gate evidence (SLM-282 recursive_core_negative; SLM-317 inconclusive with the value gate failed). There is no shared core, update mode, or specialization path to train stochastic loop counts on. SLM-139 stays closed.", + "run_id": "slm326_gate_closeout", + "schema": "slm326_lar3_gate_closeout/v1", + "status": "closeout", + "version_stamp": { + "code_commit": "480e59af94f3a1a732f2a539cbb1ae518fd540aa", + "code_dirty": false, + "components": { + "harness.experiments": "v83" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:34:15.470511+00:00" + } +} diff --git a/docs/design/iter-slm326-lar3-gate-closeout-20260725.md b/docs/design/iter-slm326-lar3-gate-closeout-20260725.md new file mode 100644 index 000000000..a8a1d0e2b --- /dev/null +++ b/docs/design/iter-slm326-lar3-gate-closeout-20260725.md @@ -0,0 +1,13 @@ +# SLM-326 / LAR3-04: LAR3 gate closeout (slm326_gate_closeout) + +Matrix set: `slm326-stochastic-loops` · Version: `slm326-v1` · Status: **closeout** +Decision: **not_authorized** — no production code. + +All of LAR3-01/02/03 are closed `not_authorized` on the same LAR3 entry-gate +evidence (SLM-282 `recursive_core_negative`; SLM-317 `inconclusive`, value +gate failed). There is no shared core, update mode, or specialization path to +train stochastic loop counts on. SLM-139 remains closed per its own rule +("remains closed until LAR3-05 disposition"). SLM-327 stays blocked. + +Reopens with the LAR3 chain when `recursive_core_positive` and a passing +repair advancement screen exist. From 50d621fe53dfbca948b293615ed16a67f6ed857e Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:36:25 -0500 Subject: [PATCH 14/18] =?UTF-8?q?SLM-327:=20close=20the=20recurrence=20lin?= =?UTF-8?q?e=20(recursive=5Fcore=5Fnegative)=20=E2=80=94=20LAR3=20closeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../iter-slm327-lar3-closeout-20260725.json | 47 +++++++++++++++++++ .../iter-slm327-lar3-closeout-20260725.md | 38 +++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 docs/design/iter-slm327-lar3-closeout-20260725.json create mode 100644 docs/design/iter-slm327-lar3-closeout-20260725.md diff --git a/docs/design/iter-slm327-lar3-closeout-20260725.json b/docs/design/iter-slm327-lar3-closeout-20260725.json new file mode 100644 index 000000000..cad33a3c4 --- /dev/null +++ b/docs/design/iter-slm327-lar3-closeout-20260725.json @@ -0,0 +1,47 @@ +{ + "claim_class": "wiring", + "consequences": { + "lar3": "recurrence line closed for this repository state; infrastructure (recursive_denoiser, telemetry, fixtures) preserved default-off", + "lar4": "remains blocked \u2014 LAR4 opens only on a powered semantic improvement that does not exist", + "production_defaults": "unchanged; no checkpoint, config, or gate change", + "slm139": "remains closed (no_supported_probabilistic_regime); nothing in this closeout reopens it" + }, + "decision": "recursive_core_negative", + "evidence": { + "lar3_closeouts": [ + "docs/design/iter-slm319-lar3-gate-closeout-20260725.json", + "docs/design/iter-slm321-lar3-gate-closeout-20260725.json", + "docs/design/iter-slm324-lar3-gate-closeout-20260725.json", + "docs/design/iter-slm326-lar3-gate-closeout-20260725.json" + ], + "slm233_recursive_core_gate_v2": { + "path": "docs/design/iter-slm233-recursive-campaign-20260724.json", + "verdict": "architecture_not_identifiable" + }, + "slm282_contraction_audit": { + "path": "docs/design/iter-slm282-recurrence-health-20260723.json", + "verdict": "recursive_core_negative" + }, + "slm317_repair_screen": { + "path": "docs/design/iter-slm317-repair-hybrid-20260724.json", + "verdict": "inconclusive (value gate failed)" + } + }, + "issue": "SLM-327", + "matrix_set": "slm327-recurrent-core-activation", + "matrix_version": "slm327-v1", + "reason": "The matched recurrent-core activation experiment cannot be run: LAR3-01..04 are all closed not_authorized, so no candidate configurations exist to advance. Independently, the recurrence line's direct evidence is negative at every measurement point: SLM-233's matched recursive-depth campaign returned architecture_not_identifiable with rsc3/rsc4 in blocked_claims; SLM-282's preregistered contraction audit returned recursive_core_negative (1/2 required seeds); SLM-230/231/232 returned stagnant/expansive_unstable/unstable; SLM-139 already closed no_supported_probabilistic_regime. Per this issue's own advancement rule, a negative closes the recurrence line and preserves infrastructure only.", + "reopening_conditions": "new evidence that the semantic floor has escaped (floor_escaped) AND a recurrence-health audit returning recursive_core_positive (>=2 seeds) AND a passing valid-state repair advancement screen", + "run_id": "slm327_lar3_closeout", + "schema": "slm327_lar3_closeout/v1", + "status": "closeout", + "version_stamp": { + "code_commit": "dc3b56ad170434ee2c5316e0384d89a5565a482d", + "code_dirty": false, + "components": { + "harness.experiments": "v83" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:35:44.352141+00:00" + } +} diff --git a/docs/design/iter-slm327-lar3-closeout-20260725.md b/docs/design/iter-slm327-lar3-closeout-20260725.md new file mode 100644 index 000000000..a5c899fd3 --- /dev/null +++ b/docs/design/iter-slm327-lar3-closeout-20260725.md @@ -0,0 +1,38 @@ +# SLM-327 / LAR3-05: Recurrence-line closeout (slm327_lar3_closeout) + +Matrix set: `slm327-recurrent-core-activation` · Version: `slm327-v1` · Status: **closeout** +Decision: **recursive_core_negative** — the recurrence line closes for this repository state; infrastructure is preserved (default-off). + +## Why the activation experiment cannot run + +LAR3-01..04 are all closed `not_authorized` (same LAR3 entry-gate evidence), so no candidate configurations exist to advance into a powered run: + +- [SLM-319 closeout](iter-slm319-lar3-gate-closeout-20260725.md) — LAR3 entry gates unmet +- [SLM-321 closeout](iter-slm321-lar3-gate-closeout-20260725.md) +- [SLM-324 closeout](iter-slm324-lar3-gate-closeout-20260725.md) +- [SLM-326 closeout](iter-slm326-lar3-gate-closeout-20260725.md) + +## Independent evidence against the line + +| Source | Verdict | +| --- | --- | +| SLM-233 matched recursive-depth campaign (`RecursiveCoreGateV2`) | `architecture_not_identifiable`; `rsc3`/`rsc4` in blocked claims | +| SLM-282 preregistered contraction audit | `recursive_core_negative` (1/2 required seeds) | +| SLM-230/231/232 observability/dynamics/z-use | `stagnant` / `expansive_unstable` / `unstable` | +| SLM-317 repair advancement screen | `inconclusive` (value gate failed, Wilson [0.0, 0.194] vs 0.05) | +| SLM-139 stochastic width | already closed `no_supported_probabilistic_regime` | + +## Consequences + +- **SLM-139:** remains closed; nothing here reopens it. +- **LAR3:** recurrence line closed for this repository state; the recursive + denoiser, telemetry, and fixtures are preserved default-off. +- **LAR4:** remains blocked — it opens only on a powered semantic + improvement that does not exist. +- **Production defaults, checkpoints, gates:** unchanged. + +## Reopening conditions + +All of: `floor_escaped` on the semantic floor gate, a recurrence-health +audit returning `recursive_core_positive` (≥2 seeds), and a passing +valid-state repair advancement screen. From 163517024c4864a7a136b9a498c8bd52db93c24a Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:38:15 -0500 Subject: [PATCH 15/18] =?UTF-8?q?SLM-329:=20close=20LAR4-01=20not=5Fauthor?= =?UTF-8?q?ized=20=E2=80=94=20LAR3=20baseline=20negative,=20LAR2=20levers?= =?UTF-8?q?=20rejected?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...er-slm329-lar4-gate-closeout-20260725.json | 51 +++++++++++++++++++ ...iter-slm329-lar4-gate-closeout-20260725.md | 17 +++++++ 2 files changed, 68 insertions(+) create mode 100644 docs/design/iter-slm329-lar4-gate-closeout-20260725.json create mode 100644 docs/design/iter-slm329-lar4-gate-closeout-20260725.md diff --git a/docs/design/iter-slm329-lar4-gate-closeout-20260725.json b/docs/design/iter-slm329-lar4-gate-closeout-20260725.json new file mode 100644 index 000000000..c2a26f1fe --- /dev/null +++ b/docs/design/iter-slm329-lar4-gate-closeout-20260725.json @@ -0,0 +1,51 @@ +{ + "claim_class": "wiring", + "decision": "not_authorized", + "downstream_effect": "SLM-331/334 stay blocked; LAR4 remains blocked per the SLM-327 closeout.", + "failed_gates": [ + "gate_lar3_05", + "gate_lar2_02", + "gate_lar2_05" + ], + "gates": { + "gate_lar2_02": { + "evidence": "docs/design/iter-slm308-distance-value-20260724.json", + "issue": "SLM-308", + "observed_outcome": "rejected at fixture budget (beam regret +0.0, rank corr +0.072 < bars)", + "passed": false, + "required_outcome": "distance-value lever adopted" + }, + "gate_lar2_05": { + "evidence": "docs/design/iter-slm314-winner-take-all-20260724.json", + "issue": "SLM-314", + "observed_outcome": "rejected (coverage gain -0.25)", + "passed": false, + "required_outcome": "WTA coverage improvement" + }, + "gate_lar3_05": { + "evidence": "docs/design/iter-slm327-lar3-closeout-20260725.json", + "issue": "SLM-327", + "observed_outcome": "recursive_core_negative; recurrence line closed", + "passed": false, + "required_outcome": "recursive_core_positive baseline" + } + }, + "issue": "SLM-329", + "matrix_set": "slm329-cross-depth-conditioning", + "matrix_version": "slm329-v1", + "production_code_added": false, + "production_default_changed": false, + "reason": "SLM-329 is blocked by LAR3-05, LAR2-02, and LAR2-05, and its own advancement rule requires clearing the LAR3 positive baseline. SLM-327 closed the recurrence line as recursive_core_negative \u2014 no LAR3 positive baseline exists. The LAR2 levers it builds on were also measured and rejected at fixture budget: SLM-308 bounded-distance value labels (beam regret +0.0 < 0.05 bar) and SLM-314 WTA multi-mode (coverage gain -0.25 < gate). No cross-depth self-conditioning work is authorized.", + "run_id": "slm329_gate_closeout", + "schema": "slm329_lar4_gate_closeout/v1", + "status": "closeout", + "version_stamp": { + "code_commit": "50d621fe53dfbca948b293615ed16a67f6ed857e", + "code_dirty": false, + "components": { + "harness.experiments": "v83" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:37:52.487967+00:00" + } +} diff --git a/docs/design/iter-slm329-lar4-gate-closeout-20260725.md b/docs/design/iter-slm329-lar4-gate-closeout-20260725.md new file mode 100644 index 000000000..8f552e0e6 --- /dev/null +++ b/docs/design/iter-slm329-lar4-gate-closeout-20260725.md @@ -0,0 +1,17 @@ +# SLM-329 / LAR4-01: LAR4 gate closeout (slm329_gate_closeout) + +Matrix set: `slm329-cross-depth-conditioning` · Version: `slm329-v1` · Status: **closeout** +Decision: **not_authorized** — no production code. + +| Gate | Issue | Required | Observed | Passed | +| --- | --- | --- | --- | --- | +| LAR3 positive baseline | SLM-327 | `recursive_core_positive` | `recursive_core_negative`; recurrence line closed ([json](iter-slm327-lar3-closeout-20260725.json)) | **False** | +| Distance-value lever | SLM-308 | adopted per preregistered bars | rejected at fixture budget (beam regret +0.0, rank corr +0.072 < bars) ([json](iter-slm308-distance-value-20260724.json)) | **False** | +| WTA multi-mode | SLM-314 | coverage improvement | rejected (coverage gain −0.25) ([json](iter-slm314-winner-take-all-20260724.json)) | **False** | + +The issue's own advancement rule ("advance only if semantic progress is +monotone and final quality clears the LAR3 positive baseline") cannot be met: +no LAR3 positive baseline exists, and both LAR2 levers it would build on +were measured and rejected at fixture budget. No production code, defaults, +or checkpoints. SLM-331/334 stay blocked; LAR4 remains blocked per the +SLM-327 closeout. From af338a2bd4967e28a760f62399340938e875cf25 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:40:04 -0500 Subject: [PATCH 16/18] =?UTF-8?q?SLM-331:=20close=20LAR4-02=20not=5Fauthor?= =?UTF-8?q?ized=20=E2=80=94=20recurrence=20line=20closed=20at=20every=20ga?= =?UTF-8?q?te?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...er-slm331-lar4-gate-closeout-20260725.json | 48 +++++++++++++++++++ ...iter-slm331-lar4-gate-closeout-20260725.md | 14 ++++++ 2 files changed, 62 insertions(+) create mode 100644 docs/design/iter-slm331-lar4-gate-closeout-20260725.json create mode 100644 docs/design/iter-slm331-lar4-gate-closeout-20260725.md diff --git a/docs/design/iter-slm331-lar4-gate-closeout-20260725.json b/docs/design/iter-slm331-lar4-gate-closeout-20260725.json new file mode 100644 index 000000000..4e7af3d4f --- /dev/null +++ b/docs/design/iter-slm331-lar4-gate-closeout-20260725.json @@ -0,0 +1,48 @@ +{ + "claim_class": "wiring", + "decision": "not_authorized", + "downstream_effect": "SLM-334 (LAR4-03) stays blocked.", + "failed_gates": [ + "gate_lar3_05", + "gate_lar4_01", + "gate_repair_value" + ], + "gates": { + "gate_lar3_05": { + "evidence": "docs/design/iter-slm327-lar3-closeout-20260725.json", + "issue": "SLM-327", + "observed_outcome": "recursive_core_negative; recurrence line closed", + "passed": false + }, + "gate_lar4_01": { + "evidence": "docs/design/iter-slm329-lar4-gate-closeout-20260725.json", + "issue": "SLM-329", + "observed_outcome": "closed not_authorized", + "passed": false + }, + "gate_repair_value": { + "evidence": "docs/design/iter-slm317-repair-hybrid-20260724.json", + "issue": "SLM-317", + "observed_outcome": "inconclusive (value gate failed)", + "passed": false + } + }, + "issue": "SLM-331", + "matrix_set": "slm331-parallel-edit-sets", + "matrix_version": "slm331-v1", + "production_code_added": false, + "production_default_changed": false, + "reason": "SLM-331 is blocked by LAR3-05, LAR4-01, and LAR2-01. The recurrence line it would accelerate with parallel latent edit slots is closed: SLM-327 recursive_core_negative, and SLM-329 (LAR4-01 cross-depth conditioning) closed not_authorized on the same evidence. The sequential repair line it would be compared against was measured and did not clear its value gate (SLM-317 inconclusive). K-slot parallel edit prediction has no authorized base architecture.", + "run_id": "slm331_gate_closeout", + "schema": "slm331_lar4_gate_closeout/v1", + "status": "closeout", + "version_stamp": { + "code_commit": "163517024c4864a7a136b9a498c8bd52db93c24a", + "code_dirty": false, + "components": { + "harness.experiments": "v83" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:40:04.396459+00:00" + } +} diff --git a/docs/design/iter-slm331-lar4-gate-closeout-20260725.md b/docs/design/iter-slm331-lar4-gate-closeout-20260725.md new file mode 100644 index 000000000..e20f2a4f9 --- /dev/null +++ b/docs/design/iter-slm331-lar4-gate-closeout-20260725.md @@ -0,0 +1,14 @@ +# SLM-331 / LAR4-02: LAR4 gate closeout (slm331_gate_closeout) + +Matrix set: `slm331-parallel-edit-sets` · Version: `slm331-v1` · Status: **closeout** +Decision: **not_authorized** — no production code. + +| Gate | Issue | Observed | Passed | +| --- | --- | --- | --- | +| LAR3-05 recurrence line | SLM-327 | `recursive_core_negative`; line closed ([json](iter-slm327-lar3-closeout-20260725.json)) | **False** | +| LAR4-01 cross-depth conditioning | SLM-329 | closed `not_authorized` ([json](iter-slm329-lar4-gate-closeout-20260725.json)) | **False** | +| Sequential repair value | SLM-317 | `inconclusive` (value gate failed) ([json](iter-slm317-repair-hybrid-20260724.json)) | **False** | + +Parallel latent edit slots would exist to accelerate a recurrence/repair +line that is closed at every measured gate. No production code, defaults, +or checkpoints. SLM-334 stays blocked. From 368dcafbd8841ce11956070695b41fdb0bbd6381 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:41:35 -0500 Subject: [PATCH 17/18] =?UTF-8?q?SLM-334:=20close=20LAR4-03=20not=5Fauthor?= =?UTF-8?q?ized=20=E2=80=94=20no=20recurrent=20core=20or=20progress=20base?= =?UTF-8?q?line?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...er-slm334-lar4-gate-closeout-20260725.json | 41 +++++++++++++++++++ ...iter-slm334-lar4-gate-closeout-20260725.md | 13 ++++++ 2 files changed, 54 insertions(+) create mode 100644 docs/design/iter-slm334-lar4-gate-closeout-20260725.json create mode 100644 docs/design/iter-slm334-lar4-gate-closeout-20260725.md diff --git a/docs/design/iter-slm334-lar4-gate-closeout-20260725.json b/docs/design/iter-slm334-lar4-gate-closeout-20260725.json new file mode 100644 index 000000000..e717c09e3 --- /dev/null +++ b/docs/design/iter-slm334-lar4-gate-closeout-20260725.json @@ -0,0 +1,41 @@ +{ + "claim_class": "wiring", + "decision": "not_authorized", + "downstream_effect": "LAR4 remains blocked per the SLM-327 closeout.", + "failed_gates": [ + "gate_lar3_05", + "gate_lar4_01" + ], + "gates": { + "gate_lar3_05": { + "evidence": "docs/design/iter-slm327-lar3-closeout-20260725.json", + "issue": "SLM-327", + "observed_outcome": "recursive_core_negative; recurrence line closed", + "passed": false + }, + "gate_lar4_01": { + "evidence": "docs/design/iter-slm329-lar4-gate-closeout-20260725.json", + "issue": "SLM-329", + "observed_outcome": "closed not_authorized", + "passed": false + } + }, + "issue": "SLM-334", + "matrix_set": "slm334-node-routing", + "matrix_version": "slm334-v1", + "production_code_added": false, + "production_default_changed": false, + "reason": "SLM-334 is blocked by LAR3-05 and LAR4-01. SLM-327 closed the recurrence line as recursive_core_negative and SLM-329 closed not_authorized, so there is no recurrent core to route per-node compute over and no LAR4-01 progress metrics to build halting on.", + "run_id": "slm334_gate_closeout", + "schema": "slm334_lar4_gate_closeout/v1", + "status": "closeout", + "version_stamp": { + "code_commit": "af338a2bd4967e28a760f62399340938e875cf25", + "code_dirty": false, + "components": { + "harness.experiments": "v83" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:41:35.330953+00:00" + } +} diff --git a/docs/design/iter-slm334-lar4-gate-closeout-20260725.md b/docs/design/iter-slm334-lar4-gate-closeout-20260725.md new file mode 100644 index 000000000..1e5f9c677 --- /dev/null +++ b/docs/design/iter-slm334-lar4-gate-closeout-20260725.md @@ -0,0 +1,13 @@ +# SLM-334 / LAR4-03: LAR4 gate closeout (slm334_gate_closeout) + +Matrix set: `slm334-node-routing` · Version: `slm334-v1` · Status: **closeout** +Decision: **not_authorized** — no production code. + +| Gate | Issue | Observed | Passed | +| --- | --- | --- | --- | +| LAR3-05 recurrent core | SLM-327 | `recursive_core_negative`; line closed ([json](iter-slm327-lar3-closeout-20260725.json)) | **False** | +| LAR4-01 progress metrics | SLM-329 | closed `not_authorized` ([json](iter-slm329-lar4-gate-closeout-20260725.json)) | **False** | + +No recurrent core exists to route per-node compute over, and no LAR4-01 +progress metrics exist to build halting on. No production code, defaults, or +checkpoints. LAR4 remains blocked per the SLM-327 closeout. From c7ff6fd1bf45501253aa246e557157d3117fc40b Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:01:56 -0500 Subject: [PATCH 18/18] SLM-338: unified evidence publisher for plan experiments (AP-037) - AbstractPlanningResultV1: canonical result object (locked campaign manifest sha, code/model/data versions, raw/constrained/repaired metric paths, plan controls, causal interventions, latency/compute breakdown, verifier gates, AgentV/human-audit refs, claim class); content sha + publication_blockers fail-closed on missing provenance, manifest hash, controls, or total latency; promotion classes additionally require all three decode paths + negative control + checkpoint. - publish_planning_result.py: one command emits canonical JSON, narrative Markdown rendered strictly from the JSON, and a model-card-ready roster row (printed, never auto-inserted); idempotent append-only disposition sections keyed by campaign hash; exit 1 with named reasons on any blocker. - 22 tests incl. MD-numbers-come-from-JSON, fail-closed, idempotency, immutability, determinism; contract doc + fixture publication; slm338 component v1. --- .../abstract-planning-ap-037-fixture.json | 134 + .../abstract-planning-ap-037-fixture.md | 43 + .../abstract-planning-evidence-contract.md | 42 + ...er-slm338-evidence-publisher-20260725.json | 46 + ...iter-slm338-evidence-publisher-20260725.md | 60 + scripts/publish_planning_result.py | 156 + .../harnesses/autoresearch/__init__.py | 1 + .../harnesses/autoresearch/planning_result.py | 435 + src/slm_training/resources/versions.json | 15814 ++++++++-------- .../planning_result/ap037_campaign.json | 89 + .../planning_result/ap037_fixture.json | 124 + tests/test_harnesses/autoresearch/__init__.py | 0 .../autoresearch/test_planning_result.py | 148 + .../test_publish_planning_result.py | 120 + 14 files changed, 9314 insertions(+), 7898 deletions(-) create mode 100644 docs/design/abstract-planning-ap-037-fixture.json create mode 100644 docs/design/abstract-planning-ap-037-fixture.md create mode 100644 docs/design/abstract-planning-evidence-contract.md create mode 100644 docs/design/iter-slm338-evidence-publisher-20260725.json create mode 100644 docs/design/iter-slm338-evidence-publisher-20260725.md create mode 100644 scripts/publish_planning_result.py create mode 100644 src/slm_training/harnesses/autoresearch/__init__.py create mode 100644 src/slm_training/harnesses/autoresearch/planning_result.py create mode 100644 tests/fixtures/planning_result/ap037_campaign.json create mode 100644 tests/fixtures/planning_result/ap037_fixture.json create mode 100644 tests/test_harnesses/autoresearch/__init__.py create mode 100644 tests/test_harnesses/autoresearch/test_planning_result.py create mode 100644 tests/test_scripts/test_publish_planning_result.py diff --git a/docs/design/abstract-planning-ap-037-fixture.json b/docs/design/abstract-planning-ap-037-fixture.json new file mode 100644 index 000000000..63faff861 --- /dev/null +++ b/docs/design/abstract-planning-ap-037-fixture.json @@ -0,0 +1,134 @@ +{ + "agentv": { + "calibration_version": "agentv-cal-v3", + "n": 24, + "path": "outputs/ap037/agentv.jsonl" + }, + "campaign_id": "ap-037-fixture", + "campaign_manifest_sha256": "00601c6d1c08dbe7ca22a3402f2c9e93d9178d9634018e0c6db29baef3215f41", + "checkpoint": { + "name": "ap037_fixture.pt", + "sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "size_bytes": 1024, + "uri": "outputs/runs/ap037_fixture/checkpoints/last.pt" + }, + "claim_class": "diagnostic", + "content_sha256": "2e683dd3d647c6e5c7a4c38ad0a2b324348c9a2134708a38fffb6e1d5df8d29c", + "controls": [ + { + "description": "Gold plan oracle ceiling.", + "kind": "oracle", + "metrics": { + "binder_reference_f1": 0.9583 + }, + "n": 24 + }, + { + "description": "Random plan negative control.", + "kind": "random", + "metrics": { + "binder_reference_f1": 0.2083 + }, + "n": 24 + }, + { + "description": "Empty plan negative control.", + "kind": "empty", + "metrics": { + "binder_reference_f1": 0.1667 + }, + "n": 24 + }, + { + "description": "Shuffled step-order negative control.", + "kind": "shuffled", + "metrics": { + "binder_reference_f1": 0.2292 + }, + "n": 24 + } + ], + "data_snapshot_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "human_audit": { + "calibration_version": "human-cal-v1", + "n": 12, + "path": "outputs/ap037/human_audit.md" + }, + "interventions": [ + { + "delta": -0.4375, + "description": "Shuffle plan step order", + "intervention_id": "shuffle-steps", + "metric": "binder_reference_f1" + } + ], + "latency": { + "generation_seconds": 41.5, + "p95_seconds": 58.2, + "plan_seconds": 3.25, + "total_seconds": 51.5, + "verification_seconds": 6.75 + }, + "metrics": [ + { + "artifact": { + "name": "raw.jsonl", + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "size_bytes": 2048, + "uri": "outputs/ap037/raw.jsonl" + }, + "binder_reference_f1": 0.4583, + "meaning_v2": 0.3125, + "n": 24, + "parse_rate": 0.625, + "path": "raw" + }, + { + "artifact": { + "name": "constrained.jsonl", + "sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "size_bytes": 2048, + "uri": "outputs/ap037/constrained.jsonl" + }, + "binder_reference_f1": 0.6667, + "meaning_v2": 0.5417, + "n": 24, + "parse_rate": 1.0, + "path": "constrained" + }, + { + "artifact": { + "name": "repaired.jsonl", + "sha256": "3333333333333333333333333333333333333333333333333333333333333333", + "size_bytes": 2048, + "uri": "outputs/ap037/repaired.jsonl" + }, + "binder_reference_f1": 0.7083, + "meaning_v2": 0.5833, + "n": 24, + "parse_rate": 1.0, + "path": "repaired" + } + ], + "schema": "AbstractPlanningResultV1", + "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "source_dirty": false, + "verifier_gates": [ + { + "gate_id": "meaning-improves", + "observed": 0.2084, + "passed": true, + "threshold": 0.01 + } + ], + "version_stamp": { + "code_commit": "368dcafbd8841ce11956070695b41fdb0bbd6381", + "code_dirty": true, + "components": { + "harness.core": "v9", + "harness.experiments.slm338_evidence_publisher": "v1" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:59:13.813116+00:00" + } +} diff --git a/docs/design/abstract-planning-ap-037-fixture.md b/docs/design/abstract-planning-ap-037-fixture.md new file mode 100644 index 000000000..d37a76096 --- /dev/null +++ b/docs/design/abstract-planning-ap-037-fixture.md @@ -0,0 +1,43 @@ +# Abstract planning result — `ap-037-fixture` + +- Schema: `AbstractPlanningResultV1` +- Claim class: **diagnostic** +- Campaign manifest sha256: `00601c6d1c08dbe7ca22a3402f2c9e93d9178d9634018e0c6db29baef3215f41` +- Source commit: `aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa` (dirty: False) +- Data snapshot sha256: `eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee` +- Checkpoint: `ap037_fixture.pt` sha256 `ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff` + +## Decode-path metrics + +| Path | n | meaning-v2 | binder F1 | parse | +| --- | --- | --- | --- | --- | +| raw | 24 | 0.3125 | 0.4583 | 0.6250 | +| constrained | 24 | 0.5417 | 0.6667 | 1.0000 | +| repaired | 24 | 0.5833 | 0.7083 | 1.0000 | + +## Plan controls + +- **oracle** (n=24): binder_reference_f1=0.9583 +- **random** (n=24): binder_reference_f1=0.2083 +- **empty** (n=24): binder_reference_f1=0.1667 +- **shuffled** (n=24): binder_reference_f1=0.2292 + +## Causal interventions + +- `shuffle-steps` — Shuffle plan step order (binder_reference_f1 delta -0.4375) + +## Latency / compute (seconds) + +| plan | generation | verification | total | p95 | +| --- | --- | --- | --- | --- | +| 3.2500 | 41.5000 | 6.7500 | 51.5000 | 58.2000 | + +## Verifier gates + +| Gate | Observed | Threshold | Passed | +| --- | --- | --- | --- | +| meaning-improves | 0.2084 | 0.0100 | **True** | + +- AgentV: `outputs/ap037/agentv.jsonl` (n=24, calibration `agentv-cal-v3`) + +- Human audit: `outputs/ap037/human_audit.md` (n=12, calibration `human-cal-v1`) diff --git a/docs/design/abstract-planning-evidence-contract.md b/docs/design/abstract-planning-evidence-contract.md new file mode 100644 index 000000000..359d1154e --- /dev/null +++ b/docs/design/abstract-planning-evidence-contract.md @@ -0,0 +1,42 @@ +# Abstract planning evidence contract (AP-037 / SLM-338) + +One canonical result object — `AbstractPlanningResultV1` +(`src/slm_training/harnesses/autoresearch/planning_result.py`) — is the single +source of truth for every plan-experiment publication. One command, +`scripts/publish_planning_result.py`, emits the canonical JSON, the narrative +Markdown (rendered from the JSON only, never hand-edited), and a +model-card-ready roster row from that one object. No manual metric copying. + +## Fail-closed publication + +`publication_blockers(result)` must be empty before anything is emitted. It +blocks on: + +- missing / `UNKNOWN` / malformed provenance (`source_commit`, + `data_snapshot_sha256`; plus a checkpoint `ArtifactRef` for promotion classes); +- missing or malformed locked campaign manifest sha256 + (`campaign_manifest_sha256`, the SLM-337 `campaign_manifest_sha256` digest of + the locked `ExperimentCampaignV1`); +- missing total latency (`latency.total_seconds` must be present and positive); +- for `promotion_candidate` / `ship_gate` claim classes: all three decode paths + (`raw`, `constrained`, `repaired`), at least one negative plan control + (`random` / `empty` / `shuffled`), and a checkpoint reference. + +`diagnostic` / `wiring` / `fixture` / `screening` results publish without the +promotion-class requirements but still fail closed on provenance, manifest +hash, and latency. + +## Disposition log + +The section below is append-only and keyed by locked campaign manifest hash; +replaying the same campaign hash is a no-op. Sections are generated by +`scripts/publish_planning_result.py --append-disposition`; never hand-edit. +Historical iter documents (`docs/design/iter-*`) stay immutable — the publisher +only writes its own artifacts. + +## Campaign `00601c6d1c08dbe7ca22a3402f2c9e93d9178d9634018e0c6db29baef3215f41` + +- Campaign id: `ap-037-fixture` +- Claim class: **diagnostic** +- Result content sha256: `2e683dd3d647c6e5c7a4c38ad0a2b324348c9a2134708a38fffb6e1d5df8d29c` +- Evidence: `docs/design/abstract-planning-ap-037-fixture.json` diff --git a/docs/design/iter-slm338-evidence-publisher-20260725.json b/docs/design/iter-slm338-evidence-publisher-20260725.json new file mode 100644 index 000000000..914ab1951 --- /dev/null +++ b/docs/design/iter-slm338-evidence-publisher-20260725.json @@ -0,0 +1,46 @@ +{ + "ap_id": "AP-037", + "artifacts": { + "canonical_json": "docs/design/abstract-planning-ap-037-fixture.json", + "disposition_doc": "docs/design/abstract-planning-evidence-contract.md", + "fixture": "tests/fixtures/planning_result/ap037_fixture.json", + "fixture_campaign": "tests/fixtures/planning_result/ap037_campaign.json", + "model_card_row": "outputs/experiments/slm338/model_card_row.md", + "narrative_markdown": "docs/design/abstract-planning-ap-037-fixture.md" + }, + "blocker_rules": [ + "missing/UNKNOWN/malformed source_commit or data_snapshot_sha256", + "missing or malformed campaign_manifest_sha256", + "missing or non-positive total latency", + "promotion classes: missing raw/constrained/repaired paths, negative plan controls, or checkpoint" + ], + "claim_class": "diagnostic", + "component": "harness.experiments.slm338_evidence_publisher v1", + "fixture_campaign_id": "ap-037-fixture", + "fixture_content_sha256": "2e683dd3d647c6e5c7a4c38ad0a2b324348c9a2134708a38fffb6e1d5df8d29c", + "fixture_manifest_sha256": "00601c6d1c08dbe7ca22a3402f2c9e93d9178d9634018e0c6db29baef3215f41", + "honesty": "Fixture-scale diagnostic publication; no checkpoint promoted, no ship claim, MODEL_CARD.md untouched (snippet only).", + "issue": "SLM-338", + "production_code_added": true, + "production_default_changed": false, + "publisher": "scripts/publish_planning_result.py", + "run_id": "slm338_evidence_publisher_fixture", + "schema": "slm338_evidence_publisher/v1", + "schema_id": "AbstractPlanningResultV1", + "schema_module": "src/slm_training/harnesses/autoresearch/planning_result.py", + "status": "published_fixture", + "tests": { + "tests/test_harnesses/autoresearch/test_planning_result.py": "13 passed", + "tests/test_scripts/test_publish_planning_result.py": "6 passed" + }, + "version_stamp": { + "code_commit": "368dcafbd8841ce11956070695b41fdb0bbd6381", + "code_dirty": true, + "components": { + "harness.core": "v9", + "harness.experiments.slm338_evidence_publisher": "v1" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T00:57:42.701306+00:00" + } +} diff --git a/docs/design/iter-slm338-evidence-publisher-20260725.md b/docs/design/iter-slm338-evidence-publisher-20260725.md new file mode 100644 index 000000000..ccea7bca1 --- /dev/null +++ b/docs/design/iter-slm338-evidence-publisher-20260725.md @@ -0,0 +1,60 @@ +# SLM-338 / AP-037: unified abstract-planning evidence publisher (slm338_evidence_publisher_fixture) + +Status: **published_fixture** · Claim class: **diagnostic** (fixture scale — no promotion or ship claim) + +## What shipped + +- `AbstractPlanningResultV1` (`src/slm_training/harnesses/autoresearch/planning_result.py`): + one frozen, content-addressed result object carrying schema id, locked campaign + manifest sha256 (SLM-337 `campaign_manifest_sha256` digest), code/model/data + provenance (source commit, dirty flag, checkpoint `ArtifactRef`, data snapshot + sha), raw/constrained/repaired metric paths (meaning-v2, binder F1, parse), + plan controls (oracle/random/empty/shuffled), causal interventions, + latency/compute breakdown (plan/generation/verification/total/p95), verifier + gates, AgentV + human-audit references, claim class, and `version_stamp`. +- `publication_blockers(result)` — fail-closed: missing/UNKNOWN provenance, + missing manifest hash, missing total latency, and (for `promotion_candidate` / + `ship_gate`) missing raw/constrained/repaired paths, negative plan controls, + or a checkpoint reference each block with a named reason. +- `scripts/publish_planning_result.py` — one command: validates blockers (exit 1 + with reasons), then emits canonical JSON (+ content sha), narrative Markdown + rendered from the JSON only, a model-card roster row (stdout / `--snippet-out`; + MODEL_CARD.md is **not** auto-edited), and an idempotent append-only + disposition section in `docs/design/abstract-planning-evidence-contract.md` + keyed by campaign hash. Historical iter docs stay immutable. +- Component `harness.experiments.slm338_evidence_publisher` v1 registered in + `versions.json`. + +## Fixture publication run + +Command (fixture `tests/fixtures/planning_result/ap037_fixture.json`, diagnostic +claim class, locked manifest sha +`00601c6d…215f41` verified against `tests/fixtures/planning_result/ap037_campaign.json`): + +```bash +python -m scripts.publish_planning_result tests/fixtures/planning_result/ap037_fixture.json \ + --output docs/design/abstract-planning-ap-037-fixture \ + --snippet-out outputs/experiments/slm338/model_card_row.md \ + --append-disposition docs/design/abstract-planning-evidence-contract.md +``` + +Artifacts: [canonical JSON](abstract-planning-ap-037-fixture.json) · +[narrative Markdown](abstract-planning-ap-037-fixture.md) · +[disposition contract](abstract-planning-evidence-contract.md) · content sha +`2e683dd3…d8d29c`. Every number in the Markdown is rendered from the JSON; +round-trip rendering is asserted deterministic in tests. + +## Tests + +- `tests/test_harnesses/autoresearch/test_planning_result.py` — 13 passed + (round-trip, content-sha determinism, each blocker class, promotion vs + diagnostic requirements, locked-manifest hash reuse). +- `tests/test_scripts/test_publish_planning_result.py` — 6 passed (end-to-end + publish, MD-from-JSON numbers, fail-closed exit 1, idempotent disposition, + `--from-dir` merge, iter-doc immutability). + +## Honesty + +Fixture-scale diagnostic wiring only. No checkpoint promoted or synced, no +production default changed, MODEL_CARD.md and README untouched (snippet printed, +not inserted). diff --git a/scripts/publish_planning_result.py b/scripts/publish_planning_result.py new file mode 100644 index 000000000..e58b10bde --- /dev/null +++ b/scripts/publish_planning_result.py @@ -0,0 +1,156 @@ +"""Publish one AbstractPlanningResultV1 as JSON + Markdown + model-card snippet (AP-037 / SLM-338). + +Reads a result JSON (or merges a harness output directory of ``*.json`` fragments +with ``--from-dir``), refuses to publish while ``publication_blockers`` is +non-empty, then emits every artifact from that one object: + +- canonical JSON (schema + content sha256) at ``.json`` +- narrative Markdown rendered from the JSON at ``.md`` +- a model-card roster row (``--snippet-out``, else stdout) — never auto-inserted +- optional append-only disposition section in + ``docs/design/abstract-planning-evidence-contract.md`` keyed by campaign hash +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from slm_training.harnesses.autoresearch.planning_result import ( + AbstractPlanningResultV1, + publication_blockers, + render_markdown, + render_model_card_row, + stamp_version, +) + + +def _load_payload(source: Path, *, from_dir: bool) -> dict[str, Any]: + if not from_dir: + return json.loads(source.read_text(encoding="utf-8")) + merged: dict[str, Any] = {} + for fragment in sorted(source.glob("*.json")): + merged.update(json.loads(fragment.read_text(encoding="utf-8"))) + if not merged: + raise ValueError(f"no JSON fragments found in {source}") + return merged + + +def _disposition_section( + payload: dict[str, Any], *, content_sha: str, evidence_path: str +) -> str: + return ( + f"## Campaign `{payload['campaign_manifest_sha256']}`\n\n" + f"- Campaign id: `{payload['campaign_id']}`\n" + f"- Claim class: **{payload['claim_class']}**\n" + f"- Result content sha256: `{content_sha}`\n" + f"- Evidence: `{evidence_path}`\n" + ) + + +def append_disposition( + doc_path: Path, payload: dict[str, Any], *, content_sha: str, evidence_path: str +) -> bool: + """Append a campaign-hash-keyed section; return True when the doc changed.""" + manifest_sha = payload["campaign_manifest_sha256"] + existing = ( + doc_path.read_text(encoding="utf-8") + if doc_path.is_file() + else ( + "# Abstract planning evidence contract\n\n" + "Append-only disposition log keyed by locked campaign manifest hash.\n" + "Sections are generated by `scripts/publish_planning_result.py`; never hand-edit.\n" + ) + ) + if f"## Campaign `{manifest_sha}`" in existing: + return False + text = ( + existing.rstrip("\n") + + "\n\n" + + _disposition_section( + payload, content_sha=content_sha, evidence_path=evidence_path + ) + ) + doc_path.parent.mkdir(parents=True, exist_ok=True) + doc_path.write_text(text, encoding="utf-8") + return True + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "source", + type=Path, + help="result JSON file (or harness output dir with --from-dir)", + ) + parser.add_argument( + "--from-dir", + action="store_true", + help="merge all *.json fragments in the source directory", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="artifact prefix (default: abstract-planning-)", + ) + parser.add_argument( + "--snippet-out", + type=Path, + default=None, + help="model-card row destination (default: stdout)", + ) + parser.add_argument( + "--append-disposition", + type=Path, + default=None, + help="append-only disposition doc to update", + ) + args = parser.parse_args(argv) + + payload = _load_payload(args.source, from_dir=args.from_dir) + result = AbstractPlanningResultV1.from_dict(payload) + if not result.version_stamp: + result = stamp_version(result) + blockers = publication_blockers(result) + if blockers: + for blocker in blockers: + print(f"publication blocked: {blocker}", file=sys.stderr) + return 1 + + canonical = result.to_dict() + canonical["content_sha256"] = result.content_sha256() + prefix = args.output or Path(f"abstract-planning-{result.campaign_id}") + prefix.parent.mkdir(parents=True, exist_ok=True) + json_path = prefix.with_suffix(".json") + md_path = prefix.with_suffix(".md") + json_path.write_text( + json.dumps(canonical, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + md_path.write_text(render_markdown(canonical), encoding="utf-8") + + row = render_model_card_row(canonical, evidence_path=str(json_path)) + "\n" + if args.snippet_out: + args.snippet_out.parent.mkdir(parents=True, exist_ok=True) + args.snippet_out.write_text(row, encoding="utf-8") + else: + sys.stdout.write(row) + + if args.append_disposition: + changed = append_disposition( + args.append_disposition, + canonical, + content_sha=canonical["content_sha256"], + evidence_path=str(json_path), + ) + print( + f"disposition {'appended' if changed else 'already present'}: {args.append_disposition}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/autoresearch/__init__.py b/src/slm_training/harnesses/autoresearch/__init__.py new file mode 100644 index 000000000..af2893a81 --- /dev/null +++ b/src/slm_training/harnesses/autoresearch/__init__.py @@ -0,0 +1 @@ +"""Autoresearch-facing harness objects (planning evidence publication).""" diff --git a/src/slm_training/harnesses/autoresearch/planning_result.py b/src/slm_training/harnesses/autoresearch/planning_result.py new file mode 100644 index 000000000..f5ce1b9e4 --- /dev/null +++ b/src/slm_training/harnesses/autoresearch/planning_result.py @@ -0,0 +1,435 @@ +"""Canonical machine-readable result object for abstract-planning experiments (AP-037 / SLM-338). + +One ``AbstractPlanningResultV1`` is the single source of truth for a planning +experiment: provenance, the locked campaign manifest digest, raw / constrained / +repaired metric paths, plan controls, causal interventions, latency, verifier +gates, audit references, and the claim class. Publication is fail-closed: +``publication_blockers`` names every missing provenance, control, or latency +field before any JSON / Markdown / model-card artifact is emitted. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import asdict, dataclass, field, replace +from typing import Any, Mapping + +from slm_training.autoresearch.experiment_campaign import ClaimClass +from slm_training.harness_core.evidence_bundle import ArtifactRef +from slm_training.lineage.records import canonical_json +from slm_training.versioning import build_version_stamp + +SCHEMA_ID = "AbstractPlanningResultV1" +COMPONENT_ID = "harness.experiments.slm338_evidence_publisher" +UNKNOWN = "UNKNOWN" +PROMOTION_CLASSES: frozenset[str] = frozenset({"promotion_candidate", "ship_gate"}) +CLAIM_CLASSES: frozenset[str] = frozenset( + ("wiring", "fixture", "diagnostic", "screening", "promotion_candidate", "ship_gate") +) +METRIC_PATHS: tuple[str, ...] = ("raw", "constrained", "repaired") +CONTROL_KINDS: tuple[str, ...] = ("oracle", "random", "empty", "shuffled") +NEGATIVE_CONTROL_KINDS: frozenset[str] = frozenset({"random", "empty", "shuffled"}) +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") + + +def _is_known(value: str | None) -> bool: + return bool(value) and value != UNKNOWN + + +@dataclass(frozen=True) +class MetricPathV1: + """Metrics for one decode path (raw / constrained / repaired).""" + + path: str + n: int + meaning_v2: float | None = None + binder_reference_f1: float | None = None + parse_rate: float | None = None + artifact: ArtifactRef | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "path": self.path, + "n": self.n, + "meaning_v2": self.meaning_v2, + "binder_reference_f1": self.binder_reference_f1, + "parse_rate": self.parse_rate, + "artifact": self.artifact.to_dict() if self.artifact else None, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "MetricPathV1": + artifact = data.get("artifact") + return cls( + path=str(data["path"]), + n=int(data["n"]), + meaning_v2=data.get("meaning_v2"), + binder_reference_f1=data.get("binder_reference_f1"), + parse_rate=data.get("parse_rate"), + artifact=ArtifactRef.from_dict(artifact) if artifact else None, + ) + + +@dataclass(frozen=True) +class PlanControlV1: + """One plan control arm (oracle / random / empty / shuffled).""" + + kind: str + n: int + metrics: Mapping[str, float] = field(default_factory=dict) + description: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "kind": self.kind, + "n": self.n, + "metrics": dict(self.metrics), + "description": self.description, + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "PlanControlV1": + return cls( + kind=str(data["kind"]), + n=int(data["n"]), + metrics={ + str(k): float(v) for k, v in dict(data.get("metrics", {})).items() + }, + description=str(data.get("description", "")), + ) + + +@dataclass(frozen=True) +class CausalInterventionV1: + """One causal intervention and its measured effect.""" + + intervention_id: str + description: str + metric: str + delta: float + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "CausalInterventionV1": + return cls( + intervention_id=str(data["intervention_id"]), + description=str(data["description"]), + metric=str(data["metric"]), + delta=float(data["delta"]), + ) + + +@dataclass(frozen=True) +class LatencyBreakdownV1: + """Latency / compute breakdown in seconds.""" + + plan_seconds: float | None = None + generation_seconds: float | None = None + verification_seconds: float | None = None + total_seconds: float | None = None + p95_seconds: float | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "LatencyBreakdownV1": + return cls( + plan_seconds=data.get("plan_seconds"), + generation_seconds=data.get("generation_seconds"), + verification_seconds=data.get("verification_seconds"), + total_seconds=data.get("total_seconds"), + p95_seconds=data.get("p95_seconds"), + ) + + +@dataclass(frozen=True) +class VerifierGateV1: + """One verifier gate outcome.""" + + gate_id: str + passed: bool + observed: float | None = None + threshold: float | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "VerifierGateV1": + return cls( + gate_id=str(data["gate_id"]), + passed=bool(data["passed"]), + observed=data.get("observed"), + threshold=data.get("threshold"), + ) + + +@dataclass(frozen=True) +class AuditReferenceV1: + """Reference to an AgentV or human-audit bundle.""" + + path: str + n: int + calibration_version: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "AuditReferenceV1": + return cls( + path=str(data["path"]), + n=int(data["n"]), + calibration_version=str(data["calibration_version"]), + ) + + +@dataclass(frozen=True) +class AbstractPlanningResultV1: + """Single source of truth for one abstract-planning experiment result.""" + + campaign_id: str + campaign_manifest_sha256: str + claim_class: ClaimClass + source_commit: str + source_dirty: bool + data_snapshot_sha256: str + checkpoint: ArtifactRef | None = None + metrics: tuple[MetricPathV1, ...] = () + controls: tuple[PlanControlV1, ...] = () + interventions: tuple[CausalInterventionV1, ...] = () + latency: LatencyBreakdownV1 = field(default_factory=LatencyBreakdownV1) + verifier_gates: tuple[VerifierGateV1, ...] = () + agentv: AuditReferenceV1 | None = None + human_audit: AuditReferenceV1 | None = None + version_stamp: Mapping[str, Any] = field(default_factory=dict) + schema: str = SCHEMA_ID + + def to_dict(self) -> dict[str, Any]: + return { + "schema": self.schema, + "campaign_id": self.campaign_id, + "campaign_manifest_sha256": self.campaign_manifest_sha256, + "claim_class": self.claim_class, + "source_commit": self.source_commit, + "source_dirty": self.source_dirty, + "data_snapshot_sha256": self.data_snapshot_sha256, + "checkpoint": self.checkpoint.to_dict() if self.checkpoint else None, + "metrics": [item.to_dict() for item in self.metrics], + "controls": [item.to_dict() for item in self.controls], + "interventions": [item.to_dict() for item in self.interventions], + "latency": self.latency.to_dict(), + "verifier_gates": [item.to_dict() for item in self.verifier_gates], + "agentv": self.agentv.to_dict() if self.agentv else None, + "human_audit": self.human_audit.to_dict() if self.human_audit else None, + "version_stamp": dict(self.version_stamp), + } + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "AbstractPlanningResultV1": + if data.get("schema") != SCHEMA_ID: + raise ValueError( + f"expected schema {SCHEMA_ID!r}, got {data.get('schema')!r}" + ) + checkpoint = data.get("checkpoint") + agentv = data.get("agentv") + human_audit = data.get("human_audit") + return cls( + campaign_id=str(data["campaign_id"]), + campaign_manifest_sha256=str(data["campaign_manifest_sha256"]), + claim_class=str(data["claim_class"]), # type: ignore[arg-type] + source_commit=str(data["source_commit"]), + source_dirty=bool(data["source_dirty"]), + data_snapshot_sha256=str(data["data_snapshot_sha256"]), + checkpoint=ArtifactRef.from_dict(checkpoint) if checkpoint else None, + metrics=tuple( + MetricPathV1.from_dict(item) for item in data.get("metrics", ()) + ), + controls=tuple( + PlanControlV1.from_dict(item) for item in data.get("controls", ()) + ), + interventions=tuple( + CausalInterventionV1.from_dict(item) + for item in data.get("interventions", ()) + ), + latency=LatencyBreakdownV1.from_dict(data.get("latency", {})), + verifier_gates=tuple( + VerifierGateV1.from_dict(item) + for item in data.get("verifier_gates", ()) + ), + agentv=AuditReferenceV1.from_dict(agentv) if agentv else None, + human_audit=AuditReferenceV1.from_dict(human_audit) + if human_audit + else None, + version_stamp=dict(data.get("version_stamp", {})), + ) + + def content_sha256(self) -> str: + """Content digest over the canonical payload, excluding the version stamp.""" + payload = self.to_dict() + payload["version_stamp"] = {} + return hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + + def metric_path(self, path: str) -> MetricPathV1 | None: + for item in self.metrics: + if item.path == path: + return item + return None + + +def stamp_version(result: AbstractPlanningResultV1) -> AbstractPlanningResultV1: + """Return a copy carrying a fresh normalized ``version_stamp`` envelope.""" + return replace( + result, version_stamp=build_version_stamp(COMPONENT_ID, "harness.core") + ) + + +def publication_blockers(result: AbstractPlanningResultV1) -> list[str]: + """Fail-closed reasons this result may not be published (empty = publishable).""" + blockers: list[str] = [] + if result.schema != SCHEMA_ID: + blockers.append(f"schema_mismatch:{result.schema}") + if result.claim_class not in CLAIM_CLASSES: + blockers.append(f"unknown_claim_class:{result.claim_class}") + if not _SHA256_RE.match(result.campaign_manifest_sha256 or ""): + blockers.append("missing_campaign_manifest_sha256") + if not _is_known(result.source_commit) or not _COMMIT_RE.match( + result.source_commit + ): + blockers.append("missing_provenance:source_commit") + if not _is_known(result.data_snapshot_sha256) or not _SHA256_RE.match( + result.data_snapshot_sha256 + ): + blockers.append("missing_provenance:data_snapshot_sha256") + if result.latency.total_seconds is None or result.latency.total_seconds <= 0: + blockers.append("missing_total_latency") + if result.claim_class in PROMOTION_CLASSES: + missing_paths = [ + path for path in METRIC_PATHS if result.metric_path(path) is None + ] + if missing_paths: + blockers.append(f"missing_metric_paths:{','.join(missing_paths)}") + control_kinds = {item.kind for item in result.controls} + if not control_kinds & NEGATIVE_CONTROL_KINDS: + blockers.append("missing_plan_controls") + if result.checkpoint is None: + blockers.append("missing_provenance:checkpoint") + return blockers + + +def _fmt(value: float | None) -> str: + return "n/a" if value is None else f"{value:.4f}" + + +def render_markdown(payload: Mapping[str, Any]) -> str: + """Render the narrative Markdown strictly from the canonical JSON payload.""" + lines: list[str] = [ + f"# Abstract planning result — `{payload['campaign_id']}`", + "", + f"- Schema: `{payload['schema']}`", + f"- Claim class: **{payload['claim_class']}**", + f"- Campaign manifest sha256: `{payload['campaign_manifest_sha256']}`", + f"- Source commit: `{payload['source_commit']}` (dirty: {payload['source_dirty']})", + f"- Data snapshot sha256: `{payload['data_snapshot_sha256']}`", + ] + checkpoint = payload.get("checkpoint") + if checkpoint: + lines.append( + f"- Checkpoint: `{checkpoint['name']}` sha256 `{checkpoint['sha256']}`" + ) + lines += [ + "", + "## Decode-path metrics", + "", + "| Path | n | meaning-v2 | binder F1 | parse |", + "| --- | --- | --- | --- | --- |", + ] + for item in payload.get("metrics", ()): + lines.append( + f"| {item['path']} | {item['n']} | {_fmt(item.get('meaning_v2'))} " + f"| {_fmt(item.get('binder_reference_f1'))} | {_fmt(item.get('parse_rate'))} |" + ) + controls = payload.get("controls", ()) + if controls: + lines += ["", "## Plan controls", ""] + for control in controls: + metrics = ", ".join( + f"{name}={_fmt(value)}" + for name, value in sorted(control["metrics"].items()) + ) + lines.append( + f"- **{control['kind']}** (n={control['n']}): {metrics or 'no metrics'}" + ) + interventions = payload.get("interventions", ()) + if interventions: + lines += ["", "## Causal interventions", ""] + for item in interventions: + lines.append( + f"- `{item['intervention_id']}` — {item['description']} " + f"({item['metric']} delta {item['delta']:+.4f})" + ) + latency = payload.get("latency", {}) + lines += [ + "", + "## Latency / compute (seconds)", + "", + "| plan | generation | verification | total | p95 |", + "| --- | --- | --- | --- | --- |", + "| {plan} | {gen} | {ver} | {total} | {p95} |".format( + plan=_fmt(latency.get("plan_seconds")), + gen=_fmt(latency.get("generation_seconds")), + ver=_fmt(latency.get("verification_seconds")), + total=_fmt(latency.get("total_seconds")), + p95=_fmt(latency.get("p95_seconds")), + ), + ] + gates = payload.get("verifier_gates", ()) + if gates: + lines += [ + "", + "## Verifier gates", + "", + "| Gate | Observed | Threshold | Passed |", + "| --- | --- | --- | --- |", + ] + for gate in gates: + lines.append( + f"| {gate['gate_id']} | {_fmt(gate.get('observed'))} " + f"| {_fmt(gate.get('threshold'))} | **{gate['passed']}** |" + ) + for label, key in (("AgentV", "agentv"), ("Human audit", "human_audit")): + ref = payload.get(key) + if ref: + lines.append( + f"\n- {label}: `{ref['path']}` (n={ref['n']}, calibration `{ref['calibration_version']}`)" + ) + return "\n".join(lines) + "\n" + + +def render_model_card_row(payload: Mapping[str, Any], *, evidence_path: str) -> str: + """Render one MODEL_CARD roster-style row from the canonical JSON payload.""" + constrained = next( + (item for item in payload.get("metrics", ()) if item["path"] == "constrained"), + None, + ) + summary = "no constrained metrics" + if constrained is not None: + summary = ( + f"constrained meaning-v2 {_fmt(constrained.get('meaning_v2'))}, " + f"binder F1 {_fmt(constrained.get('binder_reference_f1'))}, " + f"parse {_fmt(constrained.get('parse_rate'))}" + ) + honesty = "not promotable or ship" + if payload["claim_class"] in PROMOTION_CLASSES: + honesty = "promotion-class evidence; see verifier gates" + return ( + f"| {payload['campaign_id']} abstract-planning result | `{payload['campaign_id']}` " + f"| plan experiment evidence ({payload['claim_class']}) " + f"| `{evidence_path}` " + f"| {summary} — **{honesty}** |" + ) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index dadbf251a..4e4abe497 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -1,7902 +1,7920 @@ { - "schema": "version_registry/v1", - "_readme": [ - "Canonical component-version registry for the self-improving eval/smoke/checkpoint stack.", - "Contract: docs/design/version-stamp-contract.md. Enforced by scripts/verify_version_stamps.py.", - "Any change to a file under a component's `paths` must touch that component's entry in the", - "same change: bump `version` (new history entry, newest first) or append a same-version", - "history entry whose note starts with 'no-bump:' explaining why the change is behavior-neutral.", - "`paths` are repo-relative; a trailing '/' claims a directory prefix; longest prefix wins.", - "Staleness ordering is the position in `history` (newest first), never string comparison.", - "New ids and future bumps use monotonic v1, v2, ...; pre-existing native encodings are kept." - ], - "components": { - "analysis.arity.diffusion_graph": { - "version": "v1", - "kind": "metric", - "paths": [ - "src/slm_training/dsl/analysis/arity/diffusion_graph.py", - "scripts/run_quotient_diffusion_fixture.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration; wiring-first quotient-state diffusion graph diagnostics" - } - ] - }, - "config.levers": { - "version": "v20", - "kind": "harness", - "paths": [ - "src/slm_training/levers.py", - "tests/test_levers.py" - ], - "history": [ - { - "version": "v20", - "date": "2026-07-23", - "note": "Narrow the canonical Vercel evidence bundle to top-level design JSON consumed by the deployed observability reader" - }, - { - "version": "v19", - "date": "2026-07-23", - "note": "Add fail-closed CAP0, CAP1, and CAP2 lever profiles with separate teacher and distillation permission" - }, - { - "version": "v18", - "date": "2026-07-23", - "note": "Raise the canonical run cap to the permitted three minutes so changed-test CI can complete after dependency setup" - }, - { - "version": "v17", - "date": "2026-07-22", - "note": "Generate Vercel duration and committed-evidence bundle inputs from the canonical run-policy levers" - }, - { - "version": "v16", - "date": "2026-07-22", - "note": "Move changed-test parallelism into the canonical run-policy module and expose it through the single lever catalog" - }, - { - "version": "v15", - "date": "2026-07-22", - "note": "Require visible slot inventory whenever slot-aware or semantic-role decode levers are enabled" - }, - { - "version": "v14", - "date": "2026-07-22", - "note": "Declare required-slot margin dual-path after wiring the shared bias into lexer compiler selection" - }, - { - "version": "v13", - "date": "2026-07-22", - "note": "Expose the existing slot-coverage close lever on lexer restricted and tree compiler paths" - }, - { - "version": "v12", - "date": "2026-07-22", - "note": "Centralize decode-lever companion dependencies and fail before artifacts when required slot-contract or visible semantic-role configuration is absent" - }, - { - "version": "v11", - "date": "2026-07-22", - "note": "Expose active-schema visible-slot role binding on lexer compiler paths" - }, - { - "version": "v10", - "date": "2026-07-22", - "note": "Expose the existing semantic-plan typed-array nonempty margin on lexer compiler paths" - }, - { - "version": "v9", - "date": "2026-07-22", - "note": "Expose semantic-plan family scoring and margin as executable lexer compiler levers" - }, - { - "version": "v8", - "date": "2026-07-22", - "note": "Withdraw unreachable lexer root-reference identity supervision and decoding so invalid configurations fail before artifacts" - }, - { - "version": "v7", - "date": "2026-07-22", - "note": "Expose trained root-reference identity supervision and decoding on the lexer compiler path" - }, - { - "version": "v6", - "date": "2026-07-22", - "note": "Declare trained-objective dependencies for learned decode heads and reject untrained or inert configurations before execution" - }, - { - "version": "v5", - "date": "2026-07-22", - "note": "Expose root-reference arity as an executable lexer compiler lever instead of rejecting the symbol-only path" - }, - { - "version": "v4", - "date": "2026-07-21", - "note": "Expose the atomic evaluation policy preset in the canonical lever catalog" - }, - { - "version": "v3", - "date": "2026-07-22", - "note": "Unify decode-lever capability requirements and reject configurations with no executable choice or compiler path before runs start" - }, - { - "version": "v2", - "date": "2026-07-22", - "note": "Declare output-tokenizer applicability in the canonical lever catalog and fail closed on incompatible enabled levers" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "centralize run policy and expose the complete ModelBuildConfig lever catalog with context-specific defaults" - } - ] - }, - "data.corrupt.trace": { - "version": "v1", - "kind": "data_builder", - "paths": [ - "src/slm_training/data/corrupt/trace.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration; CorruptionTraceV2 severity taxonomy and fail-closed validation for SLM-120" - } - ] - }, - "data.flow.bridge_planner": { - "version": "v2", - "kind": "data_builder", - "paths": [ - "src/slm_training/data/flow/bridge_planner.py", - "tests/test_data/flow/test_bridge_planner.py" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "make every bridge transition certificate independently replayable and replace unconditional verifier acceptance with canonical validation" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-189 FFE2-01 bridge planner protocols and deterministic engine wiring/fixture" - } - ] - }, - "data.flow.legal_edit_bridge_corpus": { - "version": "v2", - "kind": "data_builder", - "paths": [ - "src/slm_training/data/flow/bridge_corpus.py", - "scripts/build_legal_edit_bridges.py", - "tests/test_data/flow/test_bridge_corpus.py", - "tests/test_scripts/test_build_legal_edit_bridges.py", - "tests/fixtures/slm196_legal_edit_bridge/", - "src/slm_training/resources/data/train/slm196_legal_edit_bridge_fixture/", - "docs/design/legal-edit-bridge-corpus.md", - "docs/design/iter-slm196-legal-edit-bridge-20260723.json", - "docs/design/iter-slm196-legal-edit-bridge-20260723.md" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "remove planner-selected edit focus and dependency fields from the model-facing row view while retaining them only as diagnostics" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial registration; SLM-196 replay-exact full-bridge rows, content-addressed live candidate sets, quality gates, and fixture builder" - } - ] - }, - "data.semantic_contrast": { - "version": "v1", - "kind": "data_builder", - "paths": [ - "src/slm_training/data/semantic_contrast/", - "scripts/build_semantic_contrasts.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SPV2-01 versioned hard-valid semantic-contrast corpus and corruption taxonomy" - } - ] - }, - "data.test_build": { - "version": "v6", - "kind": "data_builder", - "paths": [ - "scripts/build_test_data.py", - "src/slm_training/harnesses/test_data/", - "src/slm_training/resources/test_seeds.jsonl" - ], - "history": [ - { - "version": "v6", - "date": "2026-07-23", - "note": "no-bump: clarify that the preceding v6 entry was misattributed registry metadata from SLM-230 and made no test-data builder behavior change" - }, - { - "version": "v6", - "date": "2026-07-23", - "note": "pin committed evidence stamps during reproduction and exclude elapsed runtime noise from the scientific report hash" - }, - { - "version": "v5", - "date": "2026-07-22", - "note": "Make canonical held-out targets symbol-only and structurally disjoint without weakening leakage gates" - }, - { - "version": "v4", - "date": "2026-07-22", - "note": "Stamp test-data build evidence and resolve train-manifest records from the owning checkout" - }, - { - "version": "v3", - "date": "2026-07-22", - "note": "Assert the symbol-only output contract after test-target normalization and fail before artifacts when sanitization is disabled or falls back" - }, - { - "version": "v2", - "date": "2026-07-19", - "note": "apply the shared deterministic target sanitization (enforce by default) so future eval gold matches the sanitized train distribution; committed snapshots stay immutable" - }, - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration (eval/smoke suite builder; snapshot data versions remain DataStore's contract)" - } - ] - }, - "dsl.action_descriptions": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/action_descriptions.py", - "tests/test_dsl/test_action_descriptions.py" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-20", - "note": "add fixture_vectors helper for deterministic description vector generation used by SLM-176 shortlist wiring" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-174 SDE2-07 action-alias map and alias-aware description sources" - } - ] - }, - "dsl.action_shortlist": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/action_shortlist.py", - "tests/test_dsl/test_action_shortlist.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-176 P14 description-based retrieve-then-rerank policy and wiring helpers" - } - ] - }, - "dsl.operators.contracts": { - "version": "v12", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/", - "tests/test_dsl/test_operator_contracts.py" - ], - "history": [ - { - "version": "v12", - "date": "2026-07-23", - "note": "add the default-off versioned reserved operator target codec, live legal-set application boundary, and checkpoint compatibility contract" - }, - { - "version": "v11", - "date": "2026-07-23", - "note": "export replay-authoritative ordered conversation collapse decisions and reordered hard-negative evidence" - }, - { - "version": "v10", - "date": "2026-07-23", - "note": "persist bounded typed rejection applications with exact legal-set evidence for symbolic operator corpus generation" - }, - { - "version": "v9", - "date": "2026-07-23", - "note": "export conservative replayable branch merge decisions with exact structural composition and typed conflicts" - }, - { - "version": "v8", - "date": "2026-07-23", - "note": "export immutable replayable conversation state graphs, explicit history operations, and branch-local reference cloning" - }, - { - "version": "v7", - "date": "2026-07-23", - "note": "export semantic-first equivalent-outcome preference costs, sequence diagnostics, and bounded deterministic preference groups" - }, - { - "version": "v6", - "date": "2026-07-23", - "note": "export bounded exact operator legal-set enumeration, reserved typed serialization, and conservative forcing APIs" - }, - { - "version": "v5", - "date": "2026-07-23", - "note": "export the exact topology and template-alias operator context and composed library APIs" - }, - { - "version": "v4", - "date": "2026-07-23", - "note": "export the exact OpenUI core local operator family and preserve stable executor rejection codes" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "export state-bound permutation-invariant reference table and descriptor APIs" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "validate bound operator arguments against required declaration slots and reference kinds before pack execution" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial compiler-owned AST operator, typed reference, action-effect, and application proof contracts" - } - ] - }, - "dsl.operators.conversation": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/conversation.py", - "tests/test_dsl/test_operator_conversation.py" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "test exact multi-turn collapse, history refusal, cycle refusal, and equivalent reorder preservation" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial immutable conversation state DAG with exact operator replay, undo, redo, checkout, fork, and branch-local reference invalidation" - } - ] - }, - "dsl.operators.collapse": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/collapse.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial exact ordered trace collapse with typed refusal and verified reordered hard negatives" - } - ] - }, - "dsl.grammar_capabilities": { - "version": "v3", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/grammar_capabilities.py", - "tests/test_dsl/test_grammar_capabilities.py" - ], - "history": [ - { - "version": "v3", - "date": "2026-07-24", - "note": "declare typed pack-owned finite completion domains with scope fingerprints and terminal witnesses" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "add exact Lark production occurrence traces and pack-owned witness candidate and unsupported authorities" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial pack-owned exact grammar capability adapter with typed unsupported results, analyses, fingerprints, and shared conformance" - } - ] - }, - "dsl.harness_dsl": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/harness_dsl.py", - "src/slm_training/dsl/grammars/harness.lark", - "tests/test_dsl/test_harness_dsl.py", - "docs/design/dsh1-03-symbolic-harness-dsl-20260723.json", - "docs/design/dsh1-03-symbolic-harness-dsl-20260723.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial closed versioned Harness DSL with reserved CAP0 operations, typed payloads, pack validation, declared markers, artifact refs, and canonical fingerprinted framing" - } - ] - }, - "dsl.minimal_witnesses": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/minimal_witnesses.py", - "tests/test_dsl/test_minimal_witnesses.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial per-start reachable productive alternative basis with lexicographic witnesses, exact focus traces, typed gaps, and fail-closed admission" - } - ] - }, - "dsl.operators.merge": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/merge.py", - "tests/test_dsl/test_operator_merge.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial bounded three-way branch merge with fork-ref lineage, pack revalidation, deterministic replay, and typed conflict artifacts" - } - ] - }, - "dsl.operators.references": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/references.py", - "tests/test_dsl/test_operator_references.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial state-bound opaque typed reference descriptors, stable resolution failures, ordered-parent index guards, and permutation controls" - } - ] - }, - "dsl.operators.registry": { - "version": "v3", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/registry.py", - "src/slm_training/dsl/pack.py", - "tests/test_dsl/test_operator_registry.py" - ], - "history": [ - { - "version": "v3", - "date": "2026-07-24", - "note": "bind OpenUI scoped completion domains to the pack contract instead of shared decoder policy" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: populate the existing fragment-parser slot for OpenUI without changing operator registry behavior" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: add pack-owned witness candidates and unsupported alternative reasons without changing operator registry behavior" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: add an independent grammar capability authority slot without changing operator registry behavior" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "preserve stable executor rejection codes and failed preconditions through pack-owned application evidence" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial pack-owned immutable operator registry with pure apply, dry-run authority validation, typed rejection, and exact replay" - } - ] - }, - "dsl.operators.local": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/local.py", - "tests/test_dsl/test_local_operators.py" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "expose core registered entries for composition into the pack-owned topology library" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial exact OpenUI add, remove, replace, property, and child-order operator family" - } - ] - }, - "dsl.operators.legal_set": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/legal_set.py", - "tests/test_dsl/test_operator_legal_set.py" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "retain bounded deterministic rejected application samples alongside complete rejection counts" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial exact bounded operator-action enumeration with complete/partial coverage, brute-force shadow controls, and reserved hierarchical serialization" - } - ] - }, - "dsl.operators.reserved_tokens": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/reserved_tokens.py", - "tests/test_dsl/test_reserved_operator_tokens.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial default-off v1 reserved target framing with closed parsing, exact live legal-set membership, compiler application, typed defer/reject reasons, and checkpoint incompatibility guards" - } - ] - }, - "dsl.operators.preference": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/preference.py", - "tests/test_dsl/test_operator_preference.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial semantic-gated lexicographic AST and operator-sequence preference contract with simplify/expand direction and explicit defect policy" - } - ] - }, - "dsl.operators.topology": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/operators/topology.py", - "tests/test_dsl/test_topology_operators.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial exact move, reparent, wrap, unwrap, duplicate, expand, and contract operator family" - } - ] - }, - "dsl.scope_env": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/scope_env.py", - "tests/test_dsl/test_scope_env.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; Torch-free persistent typed ScopeEnv with opaque ordinal IDs, separate surface aliases, and explicit shadowing/forward-reference policies" - } - ] - }, - "dsl.symbolic_surface": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/language_contract.py", - "tests/test_dsl/test_language_contract.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial pack-authoritative SymbolicSurfacePolicyV1 with declared runtime markers, typed spans and decisions, and alpha/opaque-ID permutation controls" - } - ] - }, - "dsl.solver.topology": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/dsl/solver/topology_adapter.py", - "src/slm_training/dsl/solver/topology_solver.py", - "tests/test_dsl/test_topology_adapter.py", - "tests/test_dsl/test_topology_solver.py", - "docs/design/verified-scope-solver.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "no-bump: referenced by SLM-188 edit-algebra harness; topology edit application unchanged" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; VSS3-03 topology finite-domain adapter/solver seam (referenced by SLM-187 parity fixture; no behavior change in this commit)" - } - ] - }, - "evals.agentv": { - "version": "v4", - "kind": "harness", - "paths": [ - "src/slm_training/evals/agentv.py", - "scripts/run_agentv_eval.mjs", - "tests/test_evals/test_agentv.py" - ], - "history": [ - { - "version": "v4", - "date": "2026-07-24", - "note": "Replace the optional Node LangSmith SDK with bounded native fetch so Vercel does not bundle CLI-only dependencies" - }, - { - "version": "v3", - "date": "2026-07-24", - "note": "Forward valid W3C run traces to the pinned AgentV runner for summary-only observational telemetry" - }, - { - "version": "v2", - "date": "2026-07-24", - "note": "Make raw AgentEvals criteria code-graded authority and derive verdicts from AgentV's published artifact index" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial registration; independently enforce named case checks and attach canonical version stamps to AgentV result artifacts" - } - ] - }, - "evals.cap2_operator": { - "version": "v1", - "kind": "metric", - "paths": [ - "src/slm_training/evals/cap2_operator.py", - "src/slm_training/resources/evals/cap2_operator_v1.json", - "scripts/run_cap2_operator_eval_fixture.py", - "tests/test_evals/test_cap2_operator.py", - "tests/test_scripts/test_run_cap2_operator_eval_fixture.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: document opt-in LangSmith telemetry configuration in the shared README without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial frozen CAP2 symbolic operator suite with replayed held-out composition, confidence bounds, telemetry, and anti-cheat controls" - } - ] - }, - "evals.cap2_disposition": { - "version": "v1", - "kind": "metric", - "paths": [ - "src/slm_training/evals/cap2_disposition.py", - "scripts/publish_cap2_disposition.py", - "tests/test_evals/test_cap2_disposition.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial typed terminal CAP2 capability ledger with exact evidence identities, CERT_CAP2 rejection, and DSH4 action-distillation closure" - } - ] - }, - "evals.judge_resolution": { - "version": "v1", - "kind": "metric", - "paths": [ - "src/slm_training/evals/judge_resolution.py", - "scripts/run_judge_resolution_audit.py", - "tests/test_evals/test_judge_resolution.py", - "tests/test_scripts/test_run_judge_resolution_audit.py", - "docs/design/iter-slm185-judge-resolution-20260720.json", - "docs/design/iter-slm185-judge-resolution-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-185 FFE0-03 judge test-retest reliability, canonical-equivalence invariance, and semantic resolution floor wiring/fixture harness" - } - ] - }, - "evals.loss_suite": { - "version": "v1", - "kind": "metric", - "paths": [ - "src/slm_training/evals/loss_suites.py", - "src/slm_training/evals/loss_suite_v1.json", - "src/slm_training/evals/denoising_nll.py", - "src/slm_training/evals/emptiness_probe.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-19", - "note": "no-bump: extend emptiness probe with optional eval-only score-policy comparison; LOSS_SUITE_VERSION unchanged" - }, - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration; mirrors loss_suites.LOSS_SUITE_VERSION" - } - ] - }, - "evals.meaningful_program": { - "version": "2.13.0", - "kind": "metric", - "paths": [ - "src/slm_training/evals/meaningful_program.py" - ], - "history": [ - { - "version": "2.13.0", - "date": "2026-07-22", - "note": "Reject duplicate declared template-marker identities instead of treating set coverage as sufficient" - }, - { - "version": "2.12.0", - "date": "2026-07-21", - "note": "Reject every meaningfulness candidate containing a free-form output string under the symbol-only output contract" - }, - { - "version": "2.11.0", - "date": "2026-07-21", - "note": "E713: apply mechanical text-only routing rejection only when an explicitly requested component can directly own a required semantic role" - }, - { - "version": "2.10.0", - "date": "2026-07-21", - "note": "E712: preserve explicit component counts across arbitrary descriptive modifiers while stopping at conjunction, preposition, or component boundaries" - }, - { - "version": "2.9.0", - "date": "2026-07-21", - "note": "E699 rejection: restore pre-alias role semantics after generic placeholder/value aliases collapsed unrelated email and submit roles into RadioItem" - }, - { - "version": "2.8.0", - "date": "2026-07-21", - "note": "E699: align form-role schema aliases with the metric's accepted Input, TextArea, and Select placeholder/value properties" - }, - { - "version": "2.7.0", - "date": "2026-07-21", - "note": "E698: treat component-like adjectives as descriptive prose rather than explicit component requirements" - }, - { - "version": "2.6.0", - "date": "2026-07-21", - "note": "E685: normalize numbered visible roles to their base and align canonical tab triggers plus overview display text with public-schema properties" - }, - { - "version": "2.5.0", - "date": "2026-07-21", - "note": "E684: recognize schema plural component families through singular prose forms while preserving one container for singular compound mentions" - }, - { - "version": "2.4.0", - "date": "2026-07-21", - "note": "E680: recognize kicker and heading as schema-valid display-text roles so visible prose can occupy TextContent.text" - }, - { - "version": "2.3.0", - "date": "2026-07-21", - "note": "E674: accept public-schema aliases used by canonical fixtures: action may occupy Button.label, body may occupy description, and display value may occupy TextContent.text" - }, - { - "version": "2.2.1", - "date": "2026-07-20", - "note": "E636 r1 correction: preserve the established form-slot compatibility contract for Input, TextArea, and Select placeholder/value properties while retaining explicit rejection of Input.name" - }, - { - "version": "2.2.0", - "date": "2026-07-20", - "note": "E636: validate recognized placeholder roles against their public-schema string property instead of hard-coded component families, allowing schema-declared roles such as Modal.title while retaining Input.name rejection" - }, - { - "version": "2.1.0", - "date": "2026-07-20", - "note": "E618: _binding_check no longer falls back to the regex-based Gate.REFERENCES gate for structural (non-runtime-syntax) sources -- that fallback treated bare object-literal property keys (e.g. `src:` in a typed-array item `{src: ..., alt: ...}`) as unresolved variable references, permanently failing binding_correctness (reason `reference_graph_invalid`) for any correctly produced typed-array-of-objects prediction. The official-parser-derived unresolved/orphaned checks plus the $state/Query/Mutation dependency-graph reachability pass (already used for runtime sources, a safe no-op for structural sources) now run unconditionally instead." - }, - { - "version": "2.0.0", - "date": "2026-07-18", - "note": "initial registration; mirrors meaningful_program.METRIC_VERSION" - } - ] - }, - "evals.power_protocol": { - "version": "v7", - "kind": "metric", - "paths": [ - "src/slm_training/evals/power_protocol.py", - "tests/test_evals/test_power_protocol.py" - ], - "history": [ - { - "version": "v7", - "date": "2026-07-23", - "note": "Add exact paired-binary inference for preregistered shared-draw control and candidate arms" - }, - { - "version": "v6", - "date": "2026-07-23", - "note": "Add deterministic Holm step-down correction for prospectively declared hypothesis families" - }, - { - "version": "v5", - "date": "2026-07-23", - "note": "Reject boolean and floating-point preregistration sidedness values" - }, - { - "version": "v4", - "date": "2026-07-23", - "note": "Reject duplicate, boolean, and non-integer preregistration seed identifiers" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "Add prospective one-proportion sample-size planning with separate per-seed analyses and no observed-power inputs" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "Add configurable fail-closed Wilson intervals and canonical binomial rate-evidence metadata" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: ruff style fixes (rename ambiguous lambda variables); behavior-neutral" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-183 PQR powered cluster-aware confirmation protocol statistical utilities" - } - ] - }, - "evals.render_equivalence": { - "version": "v1", - "kind": "metric", - "paths": [ - "src/slm_training/evals/render_equivalence.py", - "tests/test_evals/test_render_equivalence.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-172 SDE2-05 tiered canonical/render-tree/visual-diff render-equivalence surrogate" - } - ] - }, - "evals.scoring": { - "version": "v22", - "kind": "metric", - "paths": [ - "src/slm_training/evals/" - ], - "history": [ - { - "version": "v22", - "date": "2026-07-23", - "note": "Add the terminal CAP2 capability disposition and fail-closed certificate decision" - }, - { - "version": "v21", - "date": "2026-07-23", - "note": "Add the frozen CAP2 operator, history, collapse, permutation, and anti-cheat scoring contract" - }, - { - "version": "v20", - "date": "2026-07-23", - "note": "Add exact paired-binary inference for governed campaign endpoint evidence" - }, - { - "version": "v19", - "date": "2026-07-23", - "note": "Add deterministic Holm family-wise correction with complete per-hypothesis evidence" - }, - { - "version": "v18", - "date": "2026-07-23", - "note": "Fail closed on non-integer prospective power sidedness" - }, - { - "version": "v17", - "date": "2026-07-23", - "note": "Fail closed on malformed prospective power seed identifiers" - }, - { - "version": "v16", - "date": "2026-07-23", - "note": "Add prospective binomial power preregistration metadata without post-hoc observed-power claims" - }, - { - "version": "v15", - "date": "2026-07-23", - "note": "Add count-provenanced configurable Wilson evidence for canonical evaluator binomial rates" - }, - { - "version": "v14", - "date": "2026-07-22", - "note": "Templatize positive SwitchItem and TabItem metric-gaming archetypes so accepted fixtures contain no free-form output strings" - }, - { - "version": "v13", - "date": "2026-07-21", - "note": "add OracleScoringReplayV1 harness and audit_gold_scoring CLI; existing scoring helpers unchanged" - }, - { - "version": "v12", - "date": "2026-07-21", - "note": "no-bump: SLM-186 adds new Goodhart canary slices to metric_gaming.py; existing slice scoring unchanged" - }, - { - "version": "v12", - "date": "2026-07-20", - "note": "wire default-off render_equivalence surrogate into task_scoreboard; RENDER_EQUIVALENCE_MODE remains off" - }, - { - "version": "v11", - "date": "2026-07-20", - "note": "add SPV2-04 dense teacher distribution mixture harness (dense_teacher_mixture.py) and fixture runner on top of the EFS3-01 state-source owner and SPV2-03 legal-set teacher traces" - }, - { - "version": "v10", - "date": "2026-07-19", - "note": "no-bump: relocate record_schema/eval_cache/score_policy to harness_core (old modules are sys.modules alias shims); suite_sharding and all scoring behavior unchanged" - }, - { - "version": "v10", - "date": "2026-07-19", - "note": "add EFS0-04 judge-independence audit scaffolding (JudgeIndependenceEvaluator, judge_audit annotation harness, export_annotations CLI) and rubric" - }, - { - "version": "v9", - "date": "2026-07-19", - "note": "add EFS2-01 X22 tree-edit beam-width × edit-depth scaling harness (TreeEditScalingConfig, run_scaling_grid, valid-state beam search) on top of quantized-energy/solver-state/verifier-cascade stack" - }, - { - "version": "v8", - "date": "2026-07-19", - "note": "add CAP4-03 quantized local-energy inference comparison harness (EnergyProblem, EnergyQuantizer, greedy vs exact Viterbi, QuantFormat wiring) on top of solver-state/verifier-cascade stack" - }, - { - "version": "v7", - "date": "2026-07-19", - "note": "add EFS3-01 solver-state supervision source comparison schema and mixer on top of verifier-cascade/score-policy/eval-cache stack" - }, - { - "version": "v6", - "date": "2026-07-19", - "note": "add cached cheap-to-expensive verifier cascade scaffolding (VerifierStageSpec, VerifierResultV1, VerifierCascade) on top of score-policy/eval-cache/trigger-telemetry stack" - }, - { - "version": "v5", - "date": "2026-07-19", - "note": "add EFS1-03 eval-only score-policy interface for constrained-decode candidate ranking on top of eval cache/trigger telemetry/metric-gaming stack" - }, - { - "version": "v4", - "date": "2026-07-19", - "note": "merge SDE3-01 eval cache and sharding on top of EFS2-02 trigger telemetry and SDE0-02 metric-gaming suite" - }, - { - "version": "v3", - "date": "2026-07-19", - "note": "add SDE3-01 content-addressed eval cache (EvalCache, EvalCacheKey, layer keys) and deterministic suite sharding (assign_example_ids, aggregate_shard_payloads)" - }, - { - "version": "v3", - "date": "2026-07-19", - "note": "merge EFS2-02 trigger telemetry harness on top of SDE0-02 metric-gaming stress suite" - }, - { - "version": "v2", - "date": "2026-07-19", - "note": "add EFS2-02 observe-only trigger telemetry harness (SearchTriggerObservationV1, TriggerObserver, compare_trigger_regimes) and wiring fixture" - }, - { - "version": "v2", - "date": "2026-07-19", - "note": "add SDE0-02 metric-gaming stress suite (metric_gaming.py) with deterministic adversarial cases scored by binding_aware_meaningful_v2" - }, - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration (catch-all for eval scoring helpers not claimed by a narrower component)" - } - ] - }, - "evals.semantic_bits": { - "version": "v1", - "kind": "metric", - "paths": [ - "src/slm_training/evals/semantic_bits.py", - "tests/test_evals/test_semantic_bits.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-186 compiler_state_conditional_bits and verified-utility efficiency helpers" - } - ] - }, - "evals.verified_utility": { - "version": "v1", - "kind": "metric", - "paths": [ - "src/slm_training/evals/verified_utility.py", - "tests/test_evals/test_verified_utility.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-186 VerifiedUtilityV1 schema, scalar/lexicographic ranking, pareto, CVaR, abstention economics, sensitivity, and canonical-equivalence helpers" - } - ] - }, - "flow.reference": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/flow/reference/" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "add SLM-190 FFE2-02 exact finite-state CTMC reference package (adapter, enumeration, generator, sampler, lumpability, trajectory, FlowTargetRowV1) and torch-free adapters" - } - ] - }, - "flow.termination": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/flow/termination.py", - "tests/test_flow/termination/test_termination_policy.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-191 FFE2-03 shared TerminationPolicy protocol, arms, and exact-CTMC sampler instrumentation" - } - ] - }, - "features.openfeature": { - "version": "v9", - "kind": "harness", - "paths": [ - "src/slm_training/features/keys.py", - "src/slm_training/features/defaults.py", - "src/slm_training/features/levers.py", - "src/slm_training/features/runtime.py", - "src/slm_training/web/observability.py", - "src/slm_training/web/app.py", - "src/slm_training/web/routes.py", - "src/apps/dashboard/src/features/keys.ts", - "src/apps/dashboard/src/features/runtime.ts", - "src/apps/dashboard/src/interpret/library.tsx", - "src/apps/dashboard/src/interpret/toolProvider.ts", - "src/apps/dashboard/src/main.tsx", - "src/apps/dashboard/src/pages/Experiments.tsx", - "src/apps/dashboard/src/pages/ExperimentFlagDetail.tsx", - "docs/design/openfeature-experiments.md" - ], - "history": [ - { - "version": "v9", - "date": "2026-07-24", - "note": "Expose AgentEvals authoritative criterion summaries beside AgentV runner status on experiment scoreboards" - }, - { - "version": "v8", - "date": "2026-07-24", - "note": "Link committed model-card parameters and checkpoint evidence to dashboard scoreboards and roster metrics" - }, - { - "version": "v7", - "date": "2026-07-24", - "note": "Expose the isolated research-harness OpenFeature capability and OFREP read endpoints beside product flags" - }, - { - "version": "v6", - "date": "2026-07-23", - "note": "no-bump: route the promotion endpoint through the frozen core so the Vercel bundle excludes offline experiment runners without changing the API contract" - }, - { - "version": "v6", - "date": "2026-07-23", - "note": "Add OpenFeature flag detail evidence, source excerpts, and outcome comparisons to the experiment dashboard" - }, - { - "version": "v5", - "date": "2026-07-23", - "note": "Use canonical OpenFeature keys in product feature metadata and bootstrap payloads" - }, - { - "version": "v4", - "date": "2026-07-23", - "note": "Expose the generated model-build OpenFeature lever registry through the experiment dashboard API" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "Accept and fail-closed validate campaign governance evidence in the promotion evaluation API" - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "no-bump: doc-only cross-link to openfeature-autoresearch-experiments.md and terminology-table row for the autoresearch flagd representation; runtime, keys, defaults, and lever registry unchanged" - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "LaunchDarkly OpenFeature provider, experiment lever registry, /api/features/levers, server-side LD browser model" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "initial OpenFeature product experiment layer: in-memory + optional PostHog providers, /api/features/bootstrap, dashboard default-renderer flag" - } - ] - }, - "gates.ship": { - "version": "openui_ship_gates_v4", - "kind": "gate", - "paths": [ - "src/slm_training/harnesses/model_build/ship_gates.py", - "src/slm_training/harness_core/gate_engine.py" - ], - "history": [ - { - "version": "openui_ship_gates_v4", - "date": "2026-07-24", - "note": "Fail closed on audited X22 suite reachability below 1.0 as a measurement-integrity failure" - }, - { - "version": "openui_ship_gates_v3", - "date": "2026-07-24", - "note": "Make raw AgentEvals assertions the canonical ship-gate authority while retaining strict integrity and runtime policy" - }, - { - "version": "openui_ship_gates_v2", - "date": "2026-07-23", - "note": "no-bump: mirror the already-versioned v2 runtime and integrity policy in the meaningful-metric provenance descriptor" - }, - { - "version": "openui_ship_gates_v2", - "date": "2026-07-23", - "note": "Fail closed on malformed measurement values and explicit decode timeouts while preserving evidence and quality categories" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-23", - "note": "no-bump: add failure-category diagnostics without changing thresholds, gate checks, or pass decisions" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-22", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.12.0 to 2.13.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.11.0 to 2.12.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.10.0 to 2.11.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.9.0 to 2.10.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.8.0 to 2.9.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.7.0 to 2.8.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.6.0 to 2.7.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.5.0 to 2.6.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.4.0 to 2.5.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.3.0 to 2.4.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.2.1 to 2.3.0; thresholds remain disabled and ship policy is unchanged" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-21", - "note": "no-bump: align MEANINGFUL_METRIC_POLICY binding_aware_meaningful_v2 provenance descriptor version from 2.1.0 to 2.2.1 to match evals.meaningful_program/METRIC_VERSION; thresholds remain None (candidate_pending_calibration), no gate relaxation" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-20", - "note": "no-bump: align MEANINGFUL_METRIC_POLICY binding_aware_meaningful_v2 provenance descriptor version from 2.0.0 to 2.1.0 to match evals.meaningful_program/METRIC_VERSION; thresholds remain None (candidate_pending_calibration), no gate relaxation" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-19", - "note": "no-bump: relocate the generic gate-check loop to harness_core.gate_engine behind a normalize_suite/default_min_n seam; policy, thresholds, and the gates.json payload stay byte-identical (pinned by tests/test_harness_core/test_gate_engine_golden.py)" - }, - { - "version": "openui_ship_gates_v1", - "date": "2026-07-18", - "note": "initial registration; mirrors MEANINGFUL_METRIC_POLICY threshold_version" - } - ] - }, - "harness.autoresearch.openfeature": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/autoresearch/openfeature.py", - "tests/test_autoresearch/test_openfeature.py", - "docs/design/openfeature-autoresearch-experiments.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "no-bump: merge with main's product experiment layer (features.openfeature); design doc renamed to openfeature-autoresearch-experiments.md, provider import unguarded now that openfeature-sdk is a core dependency; resolution behavior unchanged" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; OpenFeature (CNCF) representation of typed autoresearch experiments: flagd flag-definition export (knob flags, experiment-id variants, JsonLogic targeting) plus fail-closed ExperimentFlagProvider over ExperimentSpec knobs" - } - ] - }, - "harness.autoresearch.experiment_campaign": { - "version": "v4", - "kind": "harness", - "paths": [ - "src/slm_training/autoresearch/experiment_campaign.py", - "src/slm_training/autoresearch/schemas.py", - "src/slm_training/autoresearch/storage.py", - "src/slm_training/autoresearch/engine.py", - "src/slm_training/autoresearch/rl_gate.py", - "src/slm_training/autoresearch/__init__.py", - "tests/test_autoresearch/test_experiment_campaign.py", - "tests/test_autoresearch/test_harness.py" - ], - "history": [ - { - "version": "v4", - "date": "2026-07-24", - "note": "Require AgentEvals-authoritative ship criteria rather than a self-reported AgentV boolean for RL readiness" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "Verify authoritative artifacts, outcome-boundary event binding, portable RL evidence, and atomic campaign locks" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "Reject boolean seed identifiers in governed result evidence before promotion completeness checks" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "SLM-337 immutable preregistered campaigns, pre-start digest binding, exploratory deviations, complete promotion claims, and verified RL evidence" - } - ] - }, - "harness.core": { - "version": "v9", - "kind": "harness", - "paths": [ - "src/slm_training/harness_core/" - ], - "history": [ - { - "version": "v9", - "date": "2026-07-24", - "note": "add SLM-291 EvidenceBundleV1, pluggable content-addressed evidence store, and PreregisteredClaimBarV1" - }, - { - "version": "v8", - "date": "2026-07-23", - "note": "add SLM-193 DSL-agnostic bit-exact state, candidate, closure, and bridge cache primitives" - }, - { - "version": "v7", - "date": "2026-07-23", - "note": "Keep explicit historical result blocks visible while rejecting arbitrary nested metrics from malformed canonical records" - }, - { - "version": "v6", - "date": "2026-07-23", - "note": "Expose deterministic JSON pointers from the canonical historical experiment-record normalizer" - }, - { - "version": "v5", - "date": "2026-07-23", - "note": "Validate finite real metrics and non-negative integral evidence counts before gate comparisons" - }, - { - "version": "v4", - "date": "2026-07-23", - "note": "Return an exact evidence, integrity, quality, and runtime failure partition from the generic gate engine" - }, - { - "version": "v3", - "date": "2026-07-22", - "note": "reject canonical experiment records with missing root suites instead of promoting nested diagnostic metric blocks" - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "prefer explicit meaningful-program v1 metrics over the legacy parse-rate fallback when normalizing committed experiment records" - }, - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration; frozen DSL-agnostic core extracted structurally (versioning, lineage, checkpoint_reference, gate/promotion engines, scaling math, record_schema/eval_cache/score_policy) with behavior pinned by tests/test_harness_core" - } - ] - }, - "harness.flags": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/flags/", - "src/slm_training/harnesses/model_build/experiment_flags.py", - "docs/design/openfeature-research-levers.md", - "tests/test_flags/test_openfeature_experiments.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "SLM-342: add isolated OpenFeature-compatible research-lever evaluation, assignment metadata, and OFREP read surface" - } - ] - }, - "harness_core.evidence_bundle": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harness_core/evidence_bundle.py", - "src/slm_training/harnesses/model_build/evidence_bundle_census.py", - "scripts/verify_evidence_bundles.py", - "tests/test_harness_core/test_evidence_bundle.py", - "tests/test_scripts/test_verify_evidence_bundles.py", - "docs/design/iter-slm291-evidence-bundles-20260724.json", - "docs/design/iter-slm291-evidence-bundles-20260724.md", - "docs/design/slm291-evidence-bar-adjudications-20260724.json" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-291 content-addressed EvidenceBundleV1 + CAS store, fail-closed verifier and model-card census, PreregisteredClaimBarV1, append-only retroactive adjudications" - } - ] - }, - "harness.distill": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/distill/legal_set_kl.py", - "src/slm_training/harnesses/distill/legal_set_teacher_trace.py", - "src/slm_training/harnesses/distill/semantic_repair.py", - "scripts/run_spv2_05_semantic_repair_fixture.py", - "tests/test_harnesses/distill/test_legal_set_kl.py", - "tests/test_harnesses/distill/test_semantic_repair.py", - "docs/design/iter-spv2-05-semantic-repair-20260720.md", - "docs/design/iter-spv2-05-semantic-repair-20260720.json" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-20", - "note": "add SLM-153 SPV2-05 verifier-guided minimal semantic repair fixture wiring baseline" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-151 SPV2-03 dense legal-set distillation fixture wiring baseline" - } - ] - }, - "harness.experiments": { - "version": "v83", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/" - ], - "history": [ - { - "version": "v83", - "date": "2026-07-24", - "note": "Implement SLM-294 frontier chunked execution and scoring pass in the external-ceiling harness" - }, - { - "version": "v82", - "date": "2026-07-24", - "note": "add pinned offline-only Transformers transport for SLM-266 local teacher generation" - }, - { - "version": "v81", - "date": "2026-07-24", - "note": "no-bump: correct SLM-262 completed-job test isolation and export history metadata without changing harness behavior" - }, - { - "version": "v81", - "date": "2026-07-24", - "note": "no-bump: add SLM-262 lazy experiment exports without changing existing experiment behavior" - }, - { - "version": "v81", - "date": "2026-07-24", - "note": "require durable cross-family automatic judge evidence for SLM-266 deep admission" - }, - { - "version": "v80", - "date": "2026-07-24", - "note": "add preflighted provider executor and immutable campaign raw-attempt archive for SLM-266" - }, - { - "version": "v79", - "date": "2026-07-24", - "note": "materialize deep SLM-266 admission into canonical train snapshots with raw archive pointers" - }, - { - "version": "v78", - "date": "2026-07-24", - "note": "remove human-audit gate from SLM-266 hands-off principal admission" - }, - { - "version": "v77", - "date": "2026-07-24", - "note": "add fail-closed SLM-266 teacher-program request and admission harness" - }, - { - "version": "v76", - "date": "2026-07-24", - "note": "add deterministic SLM-207 valid-edit-flow closeout harness" - }, - { - "version": "v75", - "date": "2026-07-23", - "note": "add SLM-194 legality-neutral candidate proposal and exact-fallback amortization fixture" - }, - { - "version": "v74", - "date": "2026-07-23", - "note": "integrate SLM-193 FFE3-02 bit-exact flow-cache fixture harness on current main" - }, - { - "version": "v73", - "date": "2026-07-23", - "note": "integrate SLM-192 FFE3-01 flow-pipeline cost-profile fixture harness on current main" - }, - { - "version": "v72", - "date": "2026-07-23", - "note": "add SLM-230 bounded recurrence observability and evaluation-only exit contracts" - }, - { - "version": "v71", - "date": "2026-07-23", - "note": "publish SLM-228 SpectralDispositionV1 and guard lineage/promotion from rejected spectral mechanisms" - }, - { - "version": "v70", - "date": "2026-07-23", - "note": "add SLM-226 width and finite-size null boundary with fail-closed AbsoluteSpectralTargetGateV1" - }, - { - "version": "v69", - "date": "2026-07-23", - "note": "add SLM-220 activation-side causal restriction energy, exact and JVP estimators, null controls, and fail-closed fixture retrospective" - }, - { - "version": "v68", - "date": "2026-07-23", - "note": "add SLM-219 deterministic prefix checkpoint trajectory, canonical spectral diagnostics, pinned WeightWatcher comparison, and fail-closed warning retrospective" - }, - { - "version": "v67", - "date": "2026-07-23", - "note": "add SLM-218 cross-attention and parent-child subspace retrospective owners" - }, - { - "version": "v66", - "date": "2026-07-23", - "note": "add exact-state decision-conditioned functional spectral diagnostics for SLM-217" - }, - { - "version": "v65", - "date": "2026-07-23", - "note": "add the bounded SLM-216 fixed-token spectral regime matrix and fail-closed gate" - }, - { - "version": "v64", - "date": "2026-07-23", - "note": "publish the fail-closed SLM-213 semantic-floor claim authorization gate" - }, - { - "version": "v63", - "date": "2026-07-23", - "note": "repair canonical InsertStatement application for replay-exact SLM-196 bridge corpus construction" - }, - { - "version": "v62", - "date": "2026-07-23", - "note": "Derive promotion decisions from independently verified campaign results and plumb governance through canonical entrypoints" - }, - { - "version": "v61", - "date": "2026-07-23", - "note": "Require campaign-governed promotion, integrity-check legacy frozen manifests, and bridge the power-protocol fixture to ExperimentCampaignV1" - }, - { - "version": "v60", - "date": "2026-07-21", - "note": "no-bump: source the already-v60 two-minute fixture budget from the canonical lever registry; emitted behavior is unchanged" - }, - { - "version": "v60", - "date": "2026-07-21", - "note": "hard-cap all experiment fixture wall-budget metadata at two minutes" - }, - { - "version": "v59", - "date": "2026-07-21", - "note": "add SLM-191 FFE2-03 termination-policy protocol and fixture matrix lazy re-exports" - }, - { - "version": "v58", - "date": "2026-07-21", - "note": "add SLM-190 FFE2-02 exact CTMC reference fixture harness lazy re-exports" - }, - { - "version": "v57", - "date": "2026-07-21", - "note": "add SLM-189 FFE2-01 bridge-planner fixture harness lazy re-exports" - }, - { - "version": "v56", - "date": "2026-07-21", - "note": "add SLM-212 SDE5-05 constraint-debt routing fixture harness lazy re-exports" - }, - { - "version": "v55", - "date": "2026-07-21", - "note": "no-bump: re-export SLM-222 MuonBaselineArm/MuonBaselineReport/run_muon_baseline_fixture from experiments package; existing experiment harnesses unchanged" - }, - { - "version": "v55", - "date": "2026-07-21", - "note": "add SLM-210 SDE5-03 floor-escape matrix wiring/fixture harness extending SLM-209 debt-targeted curriculum owner" - }, - { - "version": "v54", - "date": "2026-07-21", - "note": "add SLM-215 NCS0-02 SpectralAtlasV1 wiring/fixture harness for null-calibrated checkpoint atlas and outcome joins" - }, - { - "version": "v53", - "date": "2026-07-21", - "note": "add SLM-214 NCS0-01 SpectralSnapshotV1 wiring/fixture harness with per-shape randomized-ESD null calibration" - }, - { - "version": "v52", - "date": "2026-07-21", - "note": "add SLM-188 FFE1-02 edit-algebra reachability, canonical invariance, and transition-certificate wiring/fixture harness" - }, - { - "version": "v51", - "date": "2026-07-21", - "note": "add SLM-187 FFE1-01 topology solver/runtime transition parity wiring/fixture harness" - }, - { - "version": "v50", - "date": "2026-07-21", - "note": "no-bump: add SLM-186 FFE0-04 verified-utility sub-harness; no change to existing experiment harness behavior" - }, - { - "version": "v50", - "date": "2026-07-20", - "note": "add SLM-209 SDE5-02 debt-targeted semantic exposure curriculum wiring/fixture harness" - }, - { - "version": "v49", - "date": "2026-07-20", - "note": "no-bump: SLM-185 reuses harness.experiments constants (MATRIX_SET/MATRIX_VERSION/EXPERIMENT_ID pattern) only; no harness behavior changed" - }, - { - "version": "v49", - "date": "2026-07-20", - "note": "add SLM-184 single-touch confirmation firewall and preregistered claim manifest wiring/fixture harness" - }, - { - "version": "v48", - "date": "2026-07-20", - "note": "add SLM-183 PQR powered cluster-aware confirmation protocol wiring/fixture harness" - }, - { - "version": "v47", - "date": "2026-07-20", - "note": "add SLM-176 P14 action-shortlist retrieve-then-rerank wiring/fixture harness" - }, - { - "version": "v46", - "date": "2026-07-20", - "note": "add SLM-174 SDE2-07 action-alias generalization wiring/fixture harness" - }, - { - "version": "v45", - "date": "2026-07-20", - "note": "add SLM-172 SDE2-05 render-equivalence wiring/fixture harness" - }, - { - "version": "v44", - "date": "2026-07-20", - "note": "add SLM-170 SDE2-03 exposure-targeted rare-action sampling wiring/fixture harness" - }, - { - "version": "v43", - "date": "2026-07-20", - "note": "add SLM-168 public structured contract-index pointer wiring/fixture harness" - }, - { - "version": "v42", - "date": "2026-07-20", - "note": "add SLM-167 zero-training sparse-action ceiling wiring/fixture harness" - }, - { - "version": "v41", - "date": "2026-07-20", - "note": "add SLM-166 semantic connector capacity wiring/fixture harness" - }, - { - "version": "v40", - "date": "2026-07-20", - "note": "add SLM-165 2x2x2 interaction factorial wiring/fixture harness" - }, - { - "version": "v39", - "date": "2026-07-20", - "note": "add SLM-164 targeted legal-sibling contrast margin wiring/fixture harness" - }, - { - "version": "v38", - "date": "2026-07-20", - "note": "add SLM-163 schema-description action-embedding wiring/fixture harness" - }, - { - "version": "v37", - "date": "2026-07-20", - "note": "update SLM-160 SPV4-02 disposition rationale for SLM-157 now that the fixture evidence document exists" - }, - { - "version": "v36", - "date": "2026-07-20", - "note": "add SLM-157 SPV3-04 flow / consistency / trajectory-imitation wiring/fixture harness" - }, - { - "version": "v35", - "date": "2026-07-20", - "note": "add SLM-160 SPV4-02 causal architecture disposition docs/spec harness" - }, - { - "version": "v34", - "date": "2026-07-20", - "note": "add SLM-159 SPV4-01 cross-DSL semantic-plan replication wiring/fixture harness" - }, - { - "version": "v33", - "date": "2026-07-20", - "note": "add SLM-158 SPV3-05 sequence-mixer comparison wiring/fixture harness" - }, - { - "version": "v32", - "date": "2026-07-20", - "note": "add SLM-156 SPV3-03 shared recursive SemanticPlanV1 refinement wiring/fixture harness" - }, - { - "version": "v31", - "date": "2026-07-20", - "note": "add SLM-148 SPV1-05 plan-conditioned X22 × conflict-slice staged campaign wiring/fixture harness" - }, - { - "version": "v30", - "date": "2026-07-20", - "note": "add SLM-147 SPV1-04 X22 leakage-safe retrieved-valid-AST prototype seeding harness" - }, - { - "version": "v29", - "date": "2026-07-20", - "note": "add SLM-140 EFS4-04 causal synthesis manifest, loader, script, and tests" - }, - { - "version": "v28", - "date": "2026-07-20", - "note": "add SLM-146 SPV1-03 SemanticPlanV1 plan-compiler bridge fixture wiring harness" - }, - { - "version": "v27", - "date": "2026-07-20", - "note": "add SLM-135 EFS4-01 trailed-assumptions ablation fixture wiring harness" - }, - { - "version": "v26", - "date": "2026-07-20", - "note": "add SLM-180 SDE4-02 minimum-controller-capacity fixture wiring harness" - }, - { - "version": "v25", - "date": "2026-07-20", - "note": "no-bump: replace randomized string hash with stable action-index mapping in cap2-04 fixture; matrix semantics unchanged" - }, - { - "version": "v25", - "date": "2026-07-20", - "note": "add SLM-144 SPV1-01 archetype + role-set predictor fixture wiring harness" - }, - { - "version": "v24", - "date": "2026-07-19", - "note": "add SLM-133 EFS3-06 AST-sketch dedup × choice-native retrieval factorial wiring/fixture harness" - }, - { - "version": "v23", - "date": "2026-07-19", - "note": "add SLM-130 EFS3-05 canonical AST deduplication and valid semantic-mode coverage wiring/fixture harness" - }, - { - "version": "v22", - "date": "2026-07-19", - "note": "add SLM-127 EFS3-04 contract-grounded candidate selector wiring/fixture harness" - }, - { - "version": "v21", - "date": "2026-07-19", - "note": "lazy-load B3 capacity exports so importing the experiment package and Vercel web entrypoint does not require optional torch" - }, - { - "version": "v20", - "date": "2026-07-19", - "note": "add SLM-124 EFS3-03 B3 surface-vs-choice capacity-ladder v2 manifest, fixture runner, and CLI wiring" - }, - { - "version": "v19", - "date": "2026-07-19", - "note": "add SLM-121 LDI1-02 causal PEFT FTPO manifest, fixture runner, and CLI wiring" - }, - { - "version": "v18", - "date": "2026-07-19", - "note": "add SLM-120 near-solved semantic corruption-curriculum manifest, fixture runner, and CLI wiring" - }, - { - "version": "v17", - "date": "2026-07-19", - "note": "add SLM-109 E228 exposure-ladder manifest, recipe-freeze hash, and plan/fixture CLI" - }, - { - "version": "v16", - "date": "2026-07-19", - "note": "add SLM-108 external-ceiling matrix manifest, fixture runner, and report rendering" - }, - { - "version": "v15", - "date": "2026-07-19", - "note": "no-bump: relocate efficiency_gain/scaling_fit to harness_core (old modules are sys.modules alias shims) and delegate promotion checks to harness_core.promotion_engine; HARD_CATEGORIES, the OpenUI ship-gate binding, and all public signatures unchanged" - }, - { - "version": "v15", - "date": "2026-07-19", - "note": "lazy-load torch-backed ladder re-exports so the read-only Vercel web entrypoint remains torch-free" - }, - { - "version": "v14", - "date": "2026-07-19", - "note": "fix(cap3-05): make equal-byte planning honest and reconcile cost ledger" - }, - { - "version": "v13", - "date": "2026-07-19", - "note": "add CAP3-05 (SLM-94) equal-byte width × precision ladder planner and dry-run manifest support" - }, - { - "version": "v12", - "date": "2026-07-19", - "note": "add SDE3-03 (SLM-177) proxy-metric calibration activation/budget/feature manifest dataclass, builder/validator, and CLI" - }, - { - "version": "v11", - "date": "2026-07-19", - "note": "add SDE3-04 constraint-backend benchmark manifest dataclass, builder/validator, and CLI for frozen grammar-backend comparison plan" - }, - { - "version": "v10", - "date": "2026-07-19", - "note": "add SDE4-01 (SLM-179) scaffold-distillation activation manifest dataclass, builder/validator, and CLI" - }, - { - "version": "v9", - "date": "2026-07-19", - "note": "add SDE4-04 pretrained-denoiser activation/candidate/budget manifest (wiring slice)" - }, - { - "version": "v8", - "date": "2026-07-19", - "note": "add SDE4-03 teacher-paraphrase activation/budget manifest and canonical request renderer (wiring slice)" - }, - { - "version": "v7", - "date": "2026-07-19", - "note": "add EFS2-03 conflict-slice repair harness (ConflictSliceV1, repair policies, deterministic fixtures, matched-budget comparison) on top of EFS0-05/CAP5 manifests" - }, - { - "version": "v6", - "date": "2026-07-19", - "note": "add EFS0-05 rejected-lever registry (RejectedLeverV1, ReAdjudicationRowV1, paired five-seed classification, autoresearch evidence items) on top of CAP5-02/CAP5-01 manifests" - }, - { - "version": "v5", - "date": "2026-07-19", - "note": "add CAP5-02 campaign manifest dataclass, builder/validator, and CLI for preregistered quality/cost arms" - }, - { - "version": "v4", - "date": "2026-07-19", - "note": "add GrammarProfile dataclass, manifest builder/validator, and CLI for CAP5-01 grammar/profile family summaries" - }, - { - "version": "v3", - "date": "2026-07-18", - "note": "lazy-load Torch-only equal-byte planning dependencies so the web runtime remains import-safe" - }, - { - "version": "v2", - "date": "2026-07-18", - "note": "correct equal-byte arm token budgets and fail closed on reference-only low-bit training" - }, - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration (ladder, promotion, scaling fits; verified-solver files are claimed by matrix.verified_solver)" - } - ] - }, - "harness.experiments.external_ceiling": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/models/external_scorer.py", - "scripts/run_external_ceiling.py", - "src/slm_training/harnesses/experiments/external_ceiling_matrix.py", - "tests/test_scripts/test_run_external_ceiling_frontier.py" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-24", - "note": "add arm-A tiny-baseline chunk runner, locked-threshold disposition mode with EvidenceBundleV1 plus AgentV, per-arm dtype and max_new_tokens controls" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-294 external-ceiling frontier execution (chunked resume, atomic raw JSONL, Wilson scoreboard) and scorer generate()" - } - ] - }, - "harness.experiments.ast_sketch_retrieval": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/ast_sketch_retrieval_factorial.py", - "scripts/run_ast_sketch_retrieval_factorial.py", - "tests/test_harnesses/experiments/test_ast_sketch_retrieval_factorial.py", - "tests/test_scripts/test_run_ast_sketch_retrieval_factorial.py" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-21", - "note": "no-bump: source the already-v2 wall budget from the canonical lever registry; emitted behavior is unchanged" - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "hard-cap AST-sketch retrieval fixture wall-budget metadata at two minutes" - }, - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration; SLM-133 EFS3-06 AST-sketch dedup × choice-native retrieval factorial wiring/fixture harness" - } - ] - }, - "harness.experiments.candidate_selector": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/candidate_selector.py", - "scripts/run_candidate_selector.py", - "tests/test_harnesses/experiments/test_candidate_selector.py", - "tests/test_scripts/test_run_candidate_selector.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration; SLM-127 EFS3-04 contract-grounded candidate selector wiring/fixture harness" - } - ] - }, - "harness.experiments.canonical_ast_dedup": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/canonical_ast_dedup.py", - "src/slm_training/dsl/grammar/fastpath/lattice_search.py", - "scripts/run_canonical_ast_dedup.py", - "tests/test_harnesses/experiments/test_canonical_ast_dedup.py", - "tests/test_scripts/test_run_canonical_ast_dedup.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration; SLM-130 EFS3-05 canonical AST deduplication and valid semantic-mode coverage wiring/fixture harness" - } - ] - }, - "harness.experiments.claim_manifest": { - "version": "v3", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/claim_manifest.py", - "scripts/audit_experiment_firewall.py", - "tests/test_harnesses/experiments/test_claim_manifest.py", - "tests/test_scripts/test_audit_experiment_firewall.py", - "docs/design/iter-slm184-claim-manifest-20260720.json", - "docs/design/iter-slm184-claim-manifest-20260720.md" - ], - "history": [ - { - "version": "v3", - "date": "2026-07-23", - "note": "Permit idempotent replay of an identical intact frozen manifest while rejecting mutation and tampering" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "Make frozen claim manifests create-once and verify their canonical content digest before confirmation access" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-184 single-touch confirmation firewall and preregistered claim manifest wiring/fixture harness" - } - ] - }, - "harness.experiments.semantic_floor_gate": { - "version": "v1", - "kind": "gate", - "paths": [ - "src/slm_training/harnesses/experiments/semantic_floor_gate.py", - "scripts/publish_semantic_floor_gate.py", - "tests/test_harnesses/experiments/test_semantic_floor_gate.py", - "tests/test_scripts/test_publish_semantic_floor_gate.py", - "docs/design/semantic-floor-gate-v1.json", - "docs/design/semantic-floor-gate-v1.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "normalize the component version and refresh registered identities while preserving the evidence timestamp" - }, - { - "version": "sde5-06-v1", - "date": "2026-07-23", - "note": "initial SLM-213 SemanticFloorGateV1 evidence aggregation and claim authorization" - } - ] - }, - "harness.experiments.efs4_04_causal_synthesis": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/efs4_04_causal_synthesis.py", - "scripts/synthesize_efs_campaign.py", - "tests/test_harnesses/experiments/test_efs4_04_causal_synthesis.py", - "tests/test_scripts/test_synthesize_efs_campaign.py", - "docs/design/evidence-first-semantic-slm-campaign-v1.json", - "docs/design/iter-efs4-04-causal-synthesis-20260720.json", - "docs/design/iter-efs4-04-causal-synthesis-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: make --write-default-manifest exit after writing the manifest so tests do not overwrite committed design artifacts" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: add regression tests for the campaign manifest, synthesis loader, and CLI; harness behavior unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration of SLM-140 EFS4-04 Evidence-First Semantic SLM causal synthesis manifest and loader" - } - ] - }, - "harness.experiments.sde4_02_min_controller_capacity": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/sde4_02_min_controller_capacity.py", - "scripts/run_sde4_02_min_controller_capacity.py", - "tests/test_harnesses/experiments/test_sde4_02_min_controller_capacity.py", - "tests/test_scripts/test_run_sde4_02_min_controller_capacity.py", - "docs/design/iter-sde4-02-min-controller-capacity-20260720.json", - "docs/design/iter-sde4-02-min-controller-capacity-20260720.md" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-21", - "note": "no-bump: source the already-v2 wall budget from the canonical lever registry; emitted behavior is unchanged" - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "hard-cap minimum-controller fixture wall-budget metadata at two minutes" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact timestamp after sibling fixture CLI test run; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design JSON version stamp after harness.experiments v27 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact timestamps after fixture CLI rerun; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration of SLM-180 SDE4-02 minimum-controller-capacity fixture wiring harness" - } - ] - }, - "harness.experiments.sde5_floor_escape_matrix": { - "version": "sde5-03-v1", - "kind": "harness", - "paths": [ - "tests/test_harnesses/experiments/test_sde5_floor_escape_matrix.py", - "docs/design/sde5-floor-escape-matrix-results.json", - "docs/design/sde5-floor-escape-matrix-results.md" - ], - "history": [ - { - "version": "sde5-03-v1", - "date": "2026-07-21", - "note": "initial registration; SLM-210 SDE5-03 prompt-plan × grammar-mass × high-debt exposure floor-escape matrix wiring/fixture harness" - } - ] - }, - "harness.experiments.semantic_regret": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/semantic_regret_matrix.py", - "scripts/run_semantic_regret_fixture.py", - "tests/test_harnesses/experiments/test_semantic_regret_matrix.py", - "tests/test_scripts/test_run_semantic_regret_fixture.py", - "docs/design/iter-spv0-03-semantic-regret-20260719.json", - "docs/design/iter-spv0-03-semantic-regret-20260719.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: append run-record rows to design markdown and normalize trailing whitespace in fixture runner markdown string; harness behavior, thresholds, and report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact timestamp after sibling fixture CLI test run; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design JSON version stamp after harness.experiments v27 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design JSON version stamp after harness.experiments v26 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design JSON version stamp after harness.experiments v25 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration of semantic regret decomposition fixture harness (SLM-143)" - } - ] - }, - "harness.experiments.slm135_trailed_assumptions_ablation": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm135_trailed_assumptions_ablation.py", - "scripts/run_slm135_trailed_assumptions_fixture.py", - "tests/test_harnesses/experiments/test_slm135_trailed_assumptions_ablation.py", - "tests/test_scripts/test_run_slm135_trailed_assumptions_fixture.py", - "docs/design/iter-slm135-trailed-assumptions-20260720.json", - "docs/design/iter-slm135-trailed-assumptions-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact timestamp after sibling fixture CLI test run; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact timestamp after fixture CLI rerun; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration of SLM-135 EFS4-01 trailed-assumptions ablation fixture wiring harness" - } - ] - }, - "harness.experiments.slm144_plan_predictor": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm144_plan_predictor_matrix.py", - "src/slm_training/models/semantic_plan_predictor.py", - "src/slm_training/data/semantic_plan/corpus.py", - "scripts/run_slm144_plan_predictor_fixture.py", - "tests/test_harnesses/experiments/test_slm144_plan_predictor_matrix.py", - "tests/test_models/test_semantic_plan_predictor.py", - "tests/test_scripts/test_run_slm144_plan_predictor_fixture.py", - "docs/design/iter-slm144-plan-predictor-20260720.json", - "docs/design/iter-slm144-plan-predictor-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: extend build_fixture_plan_corpus with optional root/leaf component pools; default behavior, extracted plans, and SLM-144 fixture metrics unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact timestamp after sibling fixture CLI test run; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design JSON version stamp after harness.experiments v27 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design JSON version stamp after harness.experiments v26 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact timestamps after fixture CLI rerun; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration of SLM-144 SPV1-01 archetype + role-set predictor fixture harness" - } - ] - }, - "harness.experiments.slm146_plan_compiler": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/data/semantic_plan/compiler.py", - "src/slm_training/harnesses/experiments/slm146_semantic_plan_compiler.py", - "scripts/run_slm146_semantic_plan_compiler_fixture.py", - "tests/test_data/test_semantic_plan_extraction/test_compiler.py", - "tests/test_harnesses/experiments/test_slm146_semantic_plan_compiler.py", - "tests/test_scripts/test_run_slm146_semantic_plan_compiler_fixture.py", - "docs/design/iter-slm146-semantic-plan-compiler-20260720.json", - "docs/design/iter-slm146-semantic-plan-compiler-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refine fixture CLI so plan-only mode no longer overwrites committed design artifacts; refresh fixture design JSON/Markdown" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration of SLM-146 SPV1-03 SemanticPlanV1 to compiler-choice and X22 tree-edit bridge" - } - ] - }, - "harness.experiments.slm147_x22_retrieval": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm147_x22_retrieval.py", - "scripts/run_slm147_x22_retrieval.py", - "tests/test_harnesses/experiments/test_slm147_x22_retrieval.py", - "tests/test_scripts/test_run_slm147_x22_retrieval.py", - "docs/design/iter-slm147-x22-retrieval-20260720.json", - "docs/design/iter-slm147-x22-retrieval-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamp after harness.experiments v30 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration of SLM-147 SPV1-04 X22 leakage-safe retrieved-valid-AST prototype seeding harness" - } - ] - }, - "harness.experiments.slm148_x22_conflict_campaign": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm148_x22_conflict_campaign.py", - "scripts/run_slm148_x22_conflict_campaign.py", - "tests/test_harnesses/experiments/test_slm148_x22_conflict_campaign.py", - "tests/test_scripts/test_run_slm148_x22_conflict_campaign.py", - "docs/design/iter-slm148-x22-conflict-campaign-20260720.json", - "docs/design/iter-slm148-x22-conflict-campaign-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration of SLM-148 SPV1-05 plan-conditioned X22 × conflict-slice staged campaign wiring/fixture harness" - } - ] - }, - "harness.experiments.slm155_factorization_comparison": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm155_factorization_comparison.py", - "scripts/run_slm155_factorization_comparison_fixture.py", - "scripts/run_slm297_unified_rescore.py", - "tests/test_harnesses/experiments/test_slm155_factorization_comparison.py", - "tests/test_scripts/test_run_slm155_factorization_comparison_fixture.py", - "tests/test_scripts/test_run_slm297_unified_rescore.py", - "docs/design/iter-slm155-factorization-comparison-20260720.json", - "docs/design/iter-slm155-factorization-comparison-20260720.md", - "docs/design/iter-slm297-unified-rescore-20260724.json", - "docs/design/iter-slm297-unified-rescore-20260724.md" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-24", - "note": "SLM-297: retain per-example final programs in FactorizationRecord (AR via _program_from_actions, X22 seed + program_note, hybrid source), add FinalProgramOutcomeV1 unified final-program scoring + transition/paired-table helpers, run_fixture_campaign_with_records, and the run_slm297_unified_rescore CLI; aggregate rows unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-155 SPV3-02 matched AR vs plan-conditioned X22 factorization comparison wiring/fixture harness" - } - ] - }, - "harness.experiments.slm156_plan_refinement": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm156_plan_refinement.py", - "scripts/run_slm156_plan_refinement_fixture.py", - "tests/test_harnesses/experiments/test_slm156_plan_refinement.py", - "tests/test_scripts/test_run_slm156_plan_refinement_fixture.py", - "docs/design/iter-slm156-plan-refinement-20260720.json", - "docs/design/iter-slm156-plan-refinement-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-156 SPV3-03 shared recursive SemanticPlanV1 refinement wiring/fixture harness" - } - ] - }, - "harness.experiments.slm157_flow_consistency": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm157_flow_consistency.py", - "scripts/run_slm157_flow_consistency_fixture.py", - "tests/test_harnesses/experiments/test_slm157_flow_consistency.py", - "tests/test_scripts/test_run_slm157_flow_consistency_fixture.py", - "docs/design/iter-slm157-flow-consistency-20260720.json", - "docs/design/iter-slm157-flow-consistency-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-157 SPV3-04 flow / consistency / trajectory-imitation wiring/fixture harness" - } - ] - }, - "harness.experiments.slm158_mixer_comparison": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm158_mixer_comparison.py", - "scripts/run_slm158_mixer_comparison_fixture.py", - "tests/test_harnesses/experiments/test_slm158_mixer_comparison.py", - "tests/test_scripts/test_run_slm158_mixer_comparison_fixture.py", - "docs/design/iter-slm158-mixer-comparison-20260720.json", - "docs/design/iter-slm158-mixer-comparison-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-158 SPV3-05 sequence-mixer comparison wiring/fixture harness" - } - ] - }, - "harness.experiments.slm159_cross_dsl_replication": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm159_cross_dsl_replication.py", - "scripts/run_slm159_cross_dsl_replication_fixture.py", - "tests/test_harnesses/experiments/test_slm159_cross_dsl_replication.py", - "tests/test_scripts/test_run_slm159_cross_dsl_replication_fixture.py", - "docs/design/iter-slm159-cross-dsl-replication-20260720.json", - "docs/design/iter-slm159-cross-dsl-replication-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-159 SPV4-01 cross-DSL semantic-plan replication wiring/fixture harness" - } - ] - }, - "harness.experiments.slm160_spv_disposition": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm160_spv_disposition.py", - "scripts/run_slm160_spv_disposition.py", - "tests/test_harnesses/experiments/test_slm160_spv_disposition.py", - "tests/test_scripts/test_run_slm160_spv_disposition.py", - "docs/design/iter-slm160-spv-disposition-20260720.json", - "docs/design/iter-slm160-spv-disposition-20260720.md" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v2", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v2", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v2", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v2", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v2", - "date": "2026-07-20", - "note": "update SLM-157 flow/consistency disposition rationale now that the fixture evidence document exists" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-160 SPV4-02 causal architecture disposition docs/spec harness" - } - ] - }, - "harness.experiments.slm163_schema_action_embedding": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm163_schema_action_embedding.py", - "scripts/run_slm163_schema_action_embedding_fixture.py", - "tests/test_harnesses/experiments/test_slm163_schema_action_embedding.py", - "tests/test_scripts/test_run_slm163_schema_action_embedding_fixture.py", - "docs/design/iter-slm163-schema-action-embedding-20260720.json", - "docs/design/iter-slm163-schema-action-embedding-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-163 schema-description action-embedding wiring/fixture harness" - } - ] - }, - "harness.experiments.slm164_targeted_margin": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm164_targeted_margin.py", - "scripts/run_slm164_targeted_margin_fixture.py", - "tests/test_harnesses/experiments/test_slm164_targeted_margin.py", - "tests/test_scripts/test_run_slm164_targeted_margin_fixture.py", - "docs/design/iter-slm164-targeted-margin-20260720.json", - "docs/design/iter-slm164-targeted-margin-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-164 confusion-targeted legal-sibling contrast margin wiring/fixture harness" - } - ] - }, - "harness.experiments.slm165_interaction_factorial": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm165_interaction_factorial.py", - "scripts/run_slm165_interaction_factorial_fixture.py", - "tests/test_harnesses/experiments/test_slm165_interaction_factorial.py", - "tests/test_scripts/test_run_slm165_interaction_factorial_fixture.py", - "docs/design/iter-slm165-interaction-factorial-20260720.json", - "docs/design/iter-slm165-interaction-factorial-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after commit; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-165 2x2x2 interaction factorial wiring/fixture harness" - } - ] - }, - "harness.experiments.slm166_connector_capacity": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm166_connector_capacity.py", - "scripts/run_slm166_connector_capacity_fixture.py", - "tests/test_harnesses/experiments/test_slm166_connector_capacity.py", - "tests/test_scripts/test_run_slm166_connector_capacity_fixture.py", - "docs/design/iter-slm166-connector-capacity-20260720.json", - "docs/design/iter-slm166-connector-capacity-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after commit; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-166 semantic connector capacity wiring/fixture harness" - } - ] - }, - "harness.experiments.slm167_zero_training_sparse_ceiling": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm167_zero_training_sparse_ceiling.py", - "scripts/run_slm167_zero_training_sparse_ceiling_fixture.py", - "tests/test_harnesses/experiments/test_slm167_zero_training_sparse_ceiling.py", - "tests/test_scripts/test_run_slm167_zero_training_sparse_ceiling_fixture.py", - "docs/design/iter-slm167-zero-training-sparse-ceiling-20260720.json", - "docs/design/iter-slm167-zero-training-sparse-ceiling-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after commit amend; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-167 zero-training sparse-action ceiling wiring/fixture harness" - } - ] - }, - "harness.experiments.slm168_public_structured_contract_pointer": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm168_public_structured_contract_pointer.py", - "scripts/run_slm168_public_structured_contract_pointer_fixture.py", - "tests/test_harnesses/experiments/test_slm168_public_structured_contract_pointer.py", - "tests/test_scripts/test_run_slm168_public_structured_contract_pointer_fixture.py", - "docs/design/iter-slm168-public-structured-contract-pointer-20260720.json", - "docs/design/iter-slm168-public-structured-contract-pointer-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-168 public structured contract-index pointer wiring/fixture harness" - } - ] - }, - "harness.experiments.slm170_exposure_targeted_rare_action": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm170_exposure_targeted_rare_action.py", - "scripts/run_slm170_exposure_targeted_rare_action_fixture.py", - "tests/test_harnesses/experiments/test_slm170_exposure_targeted_rare_action.py", - "tests/test_scripts/test_run_slm170_exposure_targeted_rare_action_fixture.py", - "docs/design/iter-slm170-exposure-targeted-rare-action-20260720.json", - "docs/design/iter-slm170-exposure-targeted-rare-action-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-170 SDE2-03 exposure-targeted rare-action sampling wiring/fixture harness" - } - ] - }, - "harness.experiments.slm172_render_equivalence": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm172_render_equivalence.py", - "scripts/run_slm172_render_equivalence_fixture.py", - "tests/test_harnesses/experiments/test_slm172_render_equivalence.py", - "tests/test_scripts/test_run_slm172_render_equivalence_fixture.py", - "docs/design/iter-slm172-render-equivalence-20260720.json", - "docs/design/iter-slm172-render-equivalence-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-172 SDE2-05 render-equivalence wiring/fixture harness" - } - ] - }, - "harness.experiments.slm174_action_alias_generalization": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm174_action_alias_generalization.py", - "scripts/run_slm174_action_alias_generalization_fixture.py", - "tests/test_harnesses/experiments/test_slm174_action_alias_generalization.py", - "tests/test_scripts/test_run_slm174_action_alias_generalization_fixture.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-174 SDE2-07 action-alias generalization fixture harness" - } - ] - }, - "harness.experiments.slm176_action_shortlist_rerank": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm176_action_shortlist_rerank.py", - "scripts/run_slm176_action_shortlist_rerank_fixture.py", - "tests/test_harnesses/experiments/test_slm176_action_shortlist_rerank.py", - "tests/test_scripts/test_run_slm176_action_shortlist_rerank_fixture.py", - "docs/design/iter-slm176-action-shortlist-rerank-20260720.json", - "docs/design/iter-slm176-action-shortlist-rerank-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-176 P14 action-shortlist retrieve-then-rerank fixture harness" - } - ] - }, - "harness.experiments.reserved_operator_baseline": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/reserved_operator_baseline.py", - "scripts/run_reserved_operator_baseline.py", - "tests/test_harnesses/experiments/test_reserved_operator_baseline.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial E803 capacity-matched result-only, operator-only, and operator-plus-result hashed-token experiment with causal choice and compiler-membership evidence" - } - ] - }, - "harness.experiments.slm183_power_protocol": { - "version": "v3", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm183_power_protocol.py", - "scripts/run_flow_power_protocol.py", - "tests/test_harnesses/experiments/test_slm183_power_protocol.py", - "tests/test_scripts/test_run_flow_power_protocol_fixture.py", - "docs/design/iter-slm183-power-protocol-20260720.json", - "docs/design/iter-slm183-power-protocol-20260720.md" - ], - "history": [ - { - "version": "v3", - "date": "2026-07-23", - "note": "Run exact paired control and candidate arms under a pre-start campaign lock with opt-in canonical doc writes" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "Embed the canonical ExperimentCampaignV1 preregistration with binder-reference fallback and Holm family metadata" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: add seed_contrast field to PowerProtocolReport and ruff style fixes; behavior-neutral" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-183 PQR powered cluster-aware confirmation protocol fixture harness" - } - ] - }, - "harness.experiments.slm186_verified_utility": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm186_verified_utility.py", - "scripts/run_verified_utility_audit.py", - "tests/test_harnesses/experiments/test_slm186_verified_utility.py", - "tests/test_scripts/test_run_verified_utility_audit.py", - "docs/design/iter-slm186-verified-utility-20260721.json", - "docs/design/iter-slm186-verified-utility-20260721.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-186 FFE0-04 verified-utility ladder and Goodhart canary audit wiring/fixture harness" - } - ] - }, - "harness.experiments.slm187_topology_parity": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm187_topology_parity.py", - "scripts/run_slm187_topology_parity_fixture.py", - "tests/test_harnesses/experiments/test_slm187_topology_parity.py", - "tests/test_scripts/test_run_slm187_topology_parity_fixture.py", - "docs/design/iter-slm187-topology-parity-20260721.json", - "docs/design/iter-slm187-topology-parity-20260721.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-187 FFE1-01 topology solver/runtime transition parity oracle, TopologyStateV2 carrier, and fixture harness" - } - ] - }, - "harness.experiments.slm188_edit_algebra": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm188_edit_algebra.py", - "scripts/audit_edit_reachability.py", - "tests/test_harnesses/experiments/test_slm188_edit_algebra.py", - "tests/test_scripts/test_audit_edit_reachability.py", - "docs/design/iter-slm188-edit-algebra-20260721.json", - "docs/design/iter-slm188-edit-algebra-20260721.md" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "apply missing-statement insertions with typed leaf or container payloads instead of rejecting them before dispatch" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-188 FFE1-02 edit-algebra reachability, canonical invariance, TransitionCertificateV1, and audit CLI" - } - ] - }, - "harness.experiments.slm189_bridge_planner": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm189_bridge_planner.py", - "tests/test_harnesses/experiments/test_slm189_bridge_planner.py", - "tests/test_scripts/test_run_bridge_planner_audit.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-189 FFE2-01 bridge planner fixture harness" - } - ] - }, - "harness.experiments.slm190_exact_flow": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm190_exact_flow.py", - "tests/test_harnesses/experiments/test_slm190_exact_flow.py", - "tests/test_scripts/test_run_exact_flow_fixture.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-190 FFE2-02 exact finite-state CTMC reference fixture harness" - } - ] - }, - "harness.experiments.slm191_termination_matrix": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm191_termination_matrix.py", - "tests/test_harnesses/experiments/test_slm191_termination_matrix.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-191 FFE2-03 termination-policy fixture matrix, TerminationManifestV1, and calibration instrumentation" - } - ] - }, - "harness.experiments.slm262_gpu_reference_run": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm262_gpu_reference.py", - "scripts/run_gpu_reference.py", - "tests/test_harnesses/experiments/test_slm262_gpu_reference.py", - "tests/test_scripts/test_run_gpu_reference.py", - "docs/design/gpu-reference-run.md", - "docs/design/iter-slm262-gpu-reference-run-20260721.json" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: isolate completed-job reconciliation unit test from external bucket tooling" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-262 durable GPU train checkpoint eval reference-run manifest and adapter" - } - ] - }, - "harness.experiments.slm197_direct_bridge_policy": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm197_direct_bridge_policy.py", - "src/slm_training/models/legal_edit_scorer.py", - "scripts/run_direct_bridge_policy_matrix.py", - "tests/test_models/test_legal_edit_scorer.py", - "tests/test_harnesses/experiments/test_slm197_direct_bridge_policy.py", - "tests/test_scripts/test_run_direct_bridge_policy_matrix.py", - "docs/design/iter-slm197-direct-bridge-policy-20260723.json", - "docs/design/iter-slm197-direct-bridge-policy-20260723.md", - "docs/design/iter-slm197-direct-bridge-policy-agentv-20260723/", - "docs/design/quality-experiment-matrix.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: append corrected historical E615 fixture evidence to the shared quality matrix without changing the SLM-197 scorer or experiment" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: append the SLM-236 blocked RSC4 disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: append the SLM-233 gate-bounded matched recursive campaign to the shared quality matrix without changing the SLM-197 scorer or experiment" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: append the SLM-243 measured recursive-update architecture disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: append the SLM-232 measured latent-state disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: append the SLM-231 measured recurrence-dynamics disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: append the SLM-230 measured recurrence disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: append the SLM-200 measured fixture disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: append SLM-199 measured-results pointer to the shared quality matrix without changing the SLM-197 scorer or experiment" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: append SLM-198 measured-results pointer to the shared quality matrix without changing the SLM-197 scorer or experiment" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial registration; SLM-197 exact-candidate direct policy, set-mass loss, matched time encodings, checkpoint migration, free-running decode, and honest D0-D5 fixture matrix" - } - ] - }, - "harness.experiments.slm198_bridge_curriculum": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm198_bridge_curriculum.py", - "scripts/run_bridge_curriculum_matrix.py", - "tests/test_harnesses/experiments/test_slm198_bridge_curriculum.py", - "tests/test_scripts/test_run_bridge_curriculum_matrix.py", - "docs/design/iter-slm198-bridge-curriculum-20260723.json", - "docs/design/iter-slm198-bridge-curriculum-20260723.md", - "docs/design/iter-slm198-bridge-curriculum-agentv-20260723/" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "Deduplicate per-seed manifests, retain decode trace digests instead of repeated traces, and compact deployable evidence without changing measured metrics" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial registration; SLM-198 target-first bridge curriculum manifest, deterministic scheduling features, exposure proofs, resumable sampler, and honest fixture matrix" - } - ] - }, - "harness.experiments.slm199_legal_edit_flow": { - "version": "v4", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm199_legal_edit_flow.py", - "src/slm_training/models/legal_edit_flow.py", - "src/slm_training/flow/targets.py", - "src/slm_training/flow/samplers.py", - "scripts/run_legal_edit_flow_fixture.py", - "scripts/train_legal_edit_flow.py", - "tests/test_harnesses/experiments/test_slm199_legal_edit_flow.py", - "tests/test_models/test_legal_edit_flow.py", - "tests/test_flow/test_targets.py", - "tests/test_flow/test_samplers.py", - "tests/test_scripts/test_run_legal_edit_flow_fixture.py", - "tests/test_scripts/test_vercel_bundle_policy.py", - "vercel.json", - ".vercelignore", - "docs/design/iter-slm199-legal-edit-flow-20260723.json", - "docs/design/iter-slm199-legal-edit-flow-20260723.md", - "docs/design/iter-slm199-legal-edit-flow-agentv-20260723/" - ], - "history": [ - { - "version": "v4", - "date": "2026-07-24", - "note": "no-bump: exclude the offline SLM-233 campaign runner from Vercel source upload without changing deployed web behavior" - }, - { - "version": "v4", - "date": "2026-07-23", - "note": "move nested AgentV evidence exclusion to Vercel source-upload policy after live previews proved the function-level glob did not remove those files" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "exclude SLM-243 offline matrix code from the Vercel source upload while retaining its compact dashboard result JSON" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: replace the ineffective SLM-230 function glob with a source-upload ignore for the offline corpus without changing web runtime behavior" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: exclude SLM-230 offline corpus and evidence directories from the Vercel function bundle without changing web runtime behavior" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: generalize the offline Vercel harness exclusion to SLM-199/200 and exclude the SLM-200 top-level experiment report while preserving measured experiment behavior" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: regenerate SLM-199 evidence from clean v2 code with field-level AgentV checks and stamped result artifacts" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "enforce field-level AgentV checks, restricted checkpoint loading, contained certificate failures, and canonical shared run budgets" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: compact equivalent Vercel exclusions below the 256-character schema limit after preview validation rejected the expanded spelling" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: exclude offline-only SLM-199 train and experiment modules from the Vercel runtime bundle without changing rate, sampler, or evidence behavior" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial registration; SLM-199 exact finite-graph rate targets, UNKNOWN-safe adapted bridge targets, positive-rate model, exact Gillespie oracle, bounded production sampler, and honest fixture evidence" - } - ] - }, - "harness.experiments.slm200_flow_objective_attribution": { - "version": "v4", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm200_flow_objective_attribution.py", - "scripts/run_flow_objective_attribution.py", - "tests/test_harnesses/experiments/test_slm200_flow_objective_attribution.py", - "tests/test_scripts/test_run_flow_objective_attribution.py" - ], - "history": [ - { - "version": "v4", - "date": "2026-07-23", - "note": "no-bump: make the portable-path helper recognize its stable artifact-directory marker so the existing worktree-independent contract is testable on CI" - }, - { - "version": "v4", - "date": "2026-07-23", - "note": "rewrite AgentV artifact paths and encoded projection identities to portable agentv-dir references before committing evidence" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "restrict AgentV primary-parity checks to named booleans while retaining numeric parameter and initialization evidence in the result payload" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "fix normalized version-stamp construction in the SLM-200 publisher before measured evidence generation" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial registration; SLM-200 A0-A9 matched objective registry, exact-oracle control, parity validation, powered-protocol pins, confirmation firewall, and honest fixture-screen disposition" - } - ] - }, - "harness.experiments.slm209_debt_targeted_curriculum": { - "version": "v3", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm209_debt_targeted_curriculum.py", - "scripts/run_slm209_debt_targeted_curriculum_fixture.py", - "tests/test_harnesses/experiments/test_slm209_debt_targeted_curriculum.py", - "tests/test_scripts/test_run_slm209_debt_targeted_curriculum_fixture.py", - "docs/design/iter-slm209-debt-targeted-curriculum-20260720.json", - "docs/design/iter-slm209-debt-targeted-curriculum-20260720.md" - ], - "history": [ - { - "version": "v3", - "date": "2026-07-21", - "note": "extend owner with SLM-210 SDE5FloorEscapeMatrixV1, weight-permuted exposure control, and plan/mass axis audit" - }, - { - "version": "v2", - "date": "2026-07-20", - "note": "SLM-209 make debt digests deterministic (drop timestamp); add --design-json/--design-md overrides" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-209 SDE5-02 debt-targeted semantic exposure curriculum wiring/fixture harness" - } - ] - }, - "harness.experiments.slm211_untied_output_head": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm211_untied_output_head.py", - "src/slm_training/models/blocks.py", - "src/slm_training/models/hf_denoiser.py", - "src/slm_training/runtime/cactus/__init__.py", - "tests/test_models/test_tie_output_embedding.py", - "docs/design/iter-slm211-untied-output-head-20260721.json", - "docs/design/iter-slm211-untied-output-head-20260721.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-211 SDE5-04 default-on output-head tying control, copy-init untied arm, explicit checkpoint migration, optimizer deduplication, and spectral/Cactus metadata" - } - ] - }, - "harness.experiments.slm212_debt_routing": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm212_debt_routing.py", - "tests/test_harnesses/experiments/test_slm212_debt_routing.py", - "tests/test_scripts/test_run_slm212_debt_routing_fixture.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-212 SDE5-05 constraint-debt routing over decode paths wiring/fixture harness" - } - ] - }, - "harness.experiments.slm214_spectral_snapshot": { - "version": "v4", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm214_spectral_snapshot.py", - "scripts/inspect_spectral.py", - "tests/test_harnesses/experiments/test_slm214_spectral_snapshot.py", - "tests/test_scripts/test_inspect_spectral.py", - "docs/design/iter-slm214-spectral-snapshot-20260721.json", - "docs/design/iter-slm214-spectral-snapshot-20260721.md", - ".agents/skills/autotrain/references/inspect.md" - ], - "history": [ - { - "version": "v4", - "date": "2026-07-23", - "note": "expose canonical scale-invariant spectral trap projection for SLM-219 without a parallel SVD owner" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "make Xavier and Kaiming null draws deterministic with supported generator-aware uniform sampling" - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "no-bump: update test_inspect_spectral literal to match current component version v2; fixture/report semantics unchanged" - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "SLM-211: record tie_output_embedding in SpectralSnapshotV1" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-214 NCS0-01 SpectralSnapshotV1 native-PyTorch spectral backend, role registry, tied-storage detection, and randomized-ESD null calibration" - } - ] - }, - "harness.experiments.slm215_spectral_atlas": { - "version": "v3", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm215_spectral_atlas.py", - "scripts/build_spectral_atlas.py", - "tests/test_harnesses/experiments/test_slm215_spectral_atlas.py", - "tests/test_scripts/test_build_spectral_atlas.py", - "docs/design/iter-slm215-spectral-atlas-20260721.json", - "docs/design/iter-slm215-spectral-atlas-20260721.md" - ], - "history": [ - { - "version": "v3", - "date": "2026-07-23", - "note": "no-bump: pin the custom gate-path regression to the current honest verdict" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "no-bump: assert that custom gate references and hashes identify the same loaded artifact" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "record the actual caller-supplied SemanticFloorGateV1 path in atlas manifests" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "bind NCS atlas manifests to SemanticFloorGateV1 and keep strict meaning-v2 separate from legacy meaningful rate" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-215 NCS0-02 SpectralAtlasV1 fixture harness for null-calibrated checkpoint atlas and outcome joins" - } - ] - }, - "harness.experiments.slm216_spectral_regime": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm216_spectral_regime.py", - "scripts/run_spectral_regime_matrix.py", - "tests/test_harnesses/experiments/test_slm216_spectral_regime.py", - "tests/test_scripts/test_run_spectral_regime_matrix.py", - "docs/design/iter-slm216-spectral-regime-20260723.json", - "docs/design/iter-slm216-spectral-regime-20260723.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial SLM-216 fixed-token batch/data-scale matrix and SpectralRegimeGateV1" - } - ] - }, - "harness.experiments.slm226_absolute_spectral_gate": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm226_absolute_spectral_gate.py", - "scripts/run_absolute_spectral_gate.py", - "tests/test_harnesses/experiments/test_slm226_absolute_spectral_gate.py", - "tests/test_scripts/test_run_absolute_spectral_gate.py", - "docs/design/iter-slm226-absolute-spectral-gate-20260723.json", - "docs/design/iter-slm226-absolute-spectral-gate-20260723.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: refresh measured docs from the clean provenance-pinning implementation commit" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: pin check-mode provenance to the clean evidence commit across docs-only follow-up commits" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial finite-size null and scratch-width matrix plus AbsoluteSpectralTargetGateV1" - } - ] - }, - "harness.experiments.slm228_spectral_disposition": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm228_spectral_disposition.py", - "scripts/publish_spectral_disposition.py", - "scripts/model_cycle.py", - "tests/test_harnesses/experiments/test_slm228_spectral_disposition.py", - "tests/test_scripts/test_publish_spectral_disposition.py", - "docs/design/null-calibrated-spectral-learning-disposition.json", - "docs/design/null-calibrated-spectral-learning-disposition.md", - "docs/design/research-lineage.md", - "docs/MODEL_CARD.md", - "README.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: add the SLM-294 diagnostic checkpoint row to the shared model card without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: add SLM-248 research-intake contract to shared lineage without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: cross-link the SLM-236 blocked RSC4 disposition in shared README, model-card, and lineage docs without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: add SLM-194 adapted proposal-amortization evidence to shared research lineage without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: record the SLM-234 not_authorized gate closeout in shared research lineage without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: add the SLM-233 repository-diagnostic evidence to shared research lineage without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: add the SLM-243 adapted recursive-update evidence to shared research lineage without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: add the SLM-232 bounded latent-state evidence to shared roster and lineage docs without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: add the SLM-231 bounded recurrence-dynamics evidence to shared roster, operating reference, and lineage docs without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: add the SLM-230 rejected checkpoint and adapted recurrence evidence to shared roster and lineage docs without changing the SLM-228 disposition" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: refresh the generated disposition with its clean provenance-pinning implementation commit" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial final spectral disposition, evidence hashes, narrative, and promotion guards" - } - ] - }, - "harness.experiments.slm230_recurrence_observability": { - "version": "v12", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm230_recurrence_observability.py", - "scripts/run_slm230_recurrence_observability.py", - "tests/test_harnesses/experiments/test_slm230_recurrence_observability.py", - "tests/test_scripts/test_run_slm230_recurrence_observability.py", - "docs/design/iter-slm230-recurrence-observability-20260724.json", - "docs/design/iter-slm230-recurrence-observability-20260724.md", - "docs/design/iter-slm230-recurrence-observability-agentv-20260724/" - ], - "history": [ - { - "version": "v12", - "date": "2026-07-23", - "note": "run reproduction AgentV publishing in a temporary directory so check mode never rewrites committed evidence artifacts" - }, - { - "version": "v11", - "date": "2026-07-23", - "note": "no-bump: publish the clean reproducible report and pinned AgentV bundle for the already-versioned v11 harness" - }, - { - "version": "v11", - "date": "2026-07-23", - "note": "render AgentV summaries and policy rows in canonical key order so Markdown is stable across JSON serialization" - }, - { - "version": "v10", - "date": "2026-07-23", - "note": "preserve the committed dirty-evidence policy during reproduction instead of changing the hashed allow_dirty gate in check mode" - }, - { - "version": "v9", - "date": "2026-07-23", - "note": "seed every record-depth free-running decode independently so exact reproduction is invariant to preceding process RNG state" - }, - { - "version": "v8", - "date": "2026-07-23", - "note": "retain measured CPU latency in reports while excluding nondeterministic timing fields from exact scientific reproduction hashes" - }, - { - "version": "v7", - "date": "2026-07-23", - "note": "centralize the scientific report hash so generation and validation share runtime-noise exclusions while preserving the pinned evidence stamp" - }, - { - "version": "v6", - "date": "2026-07-23", - "note": "pin committed evidence stamps during reproduction and exclude elapsed runtime noise from the scientific report hash" - }, - { - "version": "v5", - "date": "2026-07-23", - "note": "count canonical recursive_outputs calls directly so every recurrence block evaluation is charged" - }, - { - "version": "v4", - "date": "2026-07-23", - "note": "bound free-running depth evaluation to two records per split and one hashed 32-token decoder policy under the hard run cap" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "bind heldout evaluation to the committed symbol-only suite and record the canonical three-minute run cap" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "bind the runner to the clean retry checkpoint and the measured 97-record strict corpus" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial RecurrenceObservabilityV1, RecurrenceExitPolicyV1, bounded checkpoint audit, and anytime controls" - } - ] - }, - "harness.experiments.slm217_functional_spectra": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm217_functional_spectra.py", - "scripts/run_functional_spectral_fixture.py", - "tests/test_harnesses/experiments/test_slm217_functional_spectra.py", - "tests/test_scripts/test_run_functional_spectral_fixture.py", - "docs/design/iter-slm217-functional-spectra-20260723.json", - "docs/design/iter-slm217-functional-spectra-20260723.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial FunctionalSpectralSnapshotV1 exact-state contract and bounded fixture" - } - ] - }, - "harness.experiments.slm218_cross_attention_retention": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm218_cross_attention_retention.py", - "scripts/run_cross_attention_retention.py", - "tests/test_harnesses/experiments/test_slm218_cross_attention_retention.py", - "tests/test_scripts/test_run_cross_attention_retention.py", - "docs/design/iter-slm218-cross-attention-retention-20260723.json", - "docs/design/iter-slm218-cross-attention-retention-20260723.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial CrossAttentionRetentionReportV1 geometry and fail-closed family coverage" - } - ] - }, - "harness.experiments.slm219_correlation_traps": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm219_correlation_traps.py", - "scripts/collect_correlation_trap_evidence.py", - "scripts/run_correlation_trap_retrospective.py", - "tests/test_harnesses/experiments/test_slm219_correlation_traps.py", - "tests/test_scripts/test_run_correlation_trap_retrospective.py", - "docs/design/iter-slm219-correlation-trap-evidence-20260723.json", - "docs/design/iter-slm219-correlation-trap-agentv-20260723/", - "docs/design/iter-slm219-correlation-trap-20260723.json", - "docs/design/iter-slm219-correlation-trap-20260723.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial CorrelationTrapReportV1 with deterministic six-checkpoint prefix evidence, independent collapse preregistration, confirmation-timed warnings, native and WeightWatcher controls, and fail-closed decision" - } - ] - }, - "harness.experiments.slm220_causal_subspace": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm220_causal_subspace.py", - "scripts/run_causal_subspace_fixture.py", - "tests/test_harnesses/experiments/test_slm220_causal_subspace.py", - "tests/test_scripts/test_run_causal_subspace_fixture.py", - "docs/design/iter-slm220-causal-subspace-20260723.json", - "docs/design/iter-slm220-causal-subspace-20260723.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial CausalSubspaceSnapshotV1 activation-side restriction estimator and rejected fixture-only retrospective" - } - ] - }, - "harness.experiments.slm222_muon_baseline": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm222_muon_baseline.py", - "docs/design/iter-slm222-muon-baseline-20260721.json", - "docs/design/iter-slm222-muon-baseline-20260721.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-222 NCS2-03 Muon/AdamW hybrid optimizer baseline fixture harness" - } - ] - }, - "harness.experiments.slm229_looped_latent_differentiation": { - "version": "v3", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm229_looped_latent_differentiation.py", - "scripts/run_slm229_looped_latent_differentiation.py", - "tests/test_harnesses/experiments/test_slm229_looped_latent_differentiation.py", - "tests/test_scripts/test_run_slm229_looped_latent_differentiation.py", - "docs/design/iter-slm229-looped-latent-differentiation-20260721.json", - "docs/design/iter-slm229-looped-latent-differentiation-20260721.md" - ], - "history": [ - { - "version": "v3", - "date": "2026-07-23", - "note": "derive learned-latent authorization narratives from the resolved semantic floor verdict" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: add explicit design output paths so fixture tests do not rewrite canonical docs; report semantics are unchanged" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "resolve the RSC learned-latent prerequisite through the authoritative SemanticFloorGateV1 hash" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "no-bump: regenerate fixture docs after merge-conflict resolution; harness code and matrix version unchanged" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-229 RSC0-01 zero-compute looped-latent differentiation and MinimalCompilerLatentContractV1 authorization audit" - } - ] - }, - "harness.gate_census": { - "version": "v8", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/model_build/evidence_census.py", - "scripts/census_ship_evidence.py", - "tests/test_harnesses/model_build/test_evidence_census.py" - ], - "history": [ - { - "version": "v8", - "date": "2026-07-23", - "note": "Preserve root metric-block JSON pointers and supersede their legacy normalized placeholders" - }, - { - "version": "v7", - "date": "2026-07-23", - "note": "Normalize legacy root and result scoreboard pointers when superseding preserved adjudication events" - }, - { - "version": "v6", - "date": "2026-07-23", - "note": "Report executing revision and canonical historical selection in the deterministic markdown renderer" - }, - { - "version": "v5", - "date": "2026-07-23", - "note": "Append a superseding adjudication when the replayed gate policy changes for an unchanged source scoreboard" - }, - { - "version": "v4", - "date": "2026-07-23", - "note": "Exclude prior ship-gate census artifacts from recursive self-census" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "Reuse canonical historical normalization, preserve verified ledger prefixes, stamp executing code, and compute exact-count interval overlap" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "Hash-chain append-only historical adjudications with deterministic supersession fields" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "Initial deterministic no-write committed-scoreboard census and append-only adjudication schema" - } - ] - }, - "harness.experiment_feature_flags": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/model_build/feature_flags.py", - "scripts/backfill_experiment_feature_flags.py", - "src/slm_training/resources/experiment_feature_flag_history.json" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "Persist experiment feature snapshots and historical evidence by canonical OpenFeature key" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "Initial OpenFeature-backed model-build lever registry, run snapshots, and historical evidence backfill" - } - ] - }, - "harness.experiments.slm231_recurrence_dynamics": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm231_recurrence_dynamics.py", - "scripts/run_slm231_recurrence_dynamics.py", - "tests/test_harnesses/experiments/test_slm231_recurrence_dynamics.py", - "tests/test_scripts/test_run_slm231_recurrence_dynamics.py", - "docs/design/iter-slm231-recurrence-dynamics-20260724.json", - "docs/design/iter-slm231-recurrence-dynamics-20260724.md", - "docs/design/iter-slm231-recurrence-dynamics-agentv-20260724/" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: publish the clean reproducible SLM-231 report and AgentV bundle generated against implementation commit 9d22aa81" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial residual-correct increment, composite, trajectory-product, FTLE, and cross-depth alignment diagnostics with synthetic, fixture, and bounded-checkpoint evidence" - } - ] - }, - "harness.experiments.slm232_latent_state_use": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm232_latent_state_use.py", - "scripts/run_slm232_latent_state_use.py", - "tests/test_harnesses/experiments/test_slm232_latent_state_use.py", - "tests/test_scripts/test_run_slm232_latent_state_use.py", - "docs/design/iter-slm232-latent-state-use-20260724.json", - "docs/design/iter-slm232-latent-state-use-20260724.md", - "docs/design/iter-slm232-latent-state-use-agentv-20260724/" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: publish the clean reproducible SLM-232 report and AgentV bundle generated against implementation commit 051c1c5d" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial bounded z-state rank, context decomposition, causal ablation, matched y-only/nuisance control, prior-join, and fail-closed disposition gate" - } - ] - }, - "harness.experiments.slm243_recursive_update_gate": { - "version": "v3", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm243_recursive_update_gate.py", - "scripts/run_slm243_recursive_update_gate.py", - "tests/test_harnesses/experiments/test_slm243_recursive_update_gate.py", - "tests/test_scripts/test_run_slm243_recursive_update_gate.py", - "docs/design/iter-slm243-recursive-update-gate-20260724.json", - "docs/design/iter-slm243-recursive-update-gate-20260724.md", - "docs/design/iter-slm243-recursive-update-gate-agentv-20260724/" - ], - "history": [ - { - "version": "v3", - "date": "2026-07-23", - "note": "no-bump: publish the compact dashboard report and hash-bound raw matrix generated cleanly against implementation commit 62726d66" - }, - { - "version": "v3", - "date": "2026-07-23", - "note": "split full 90-cell telemetry into a hash-bound raw matrix under the Vercel-excluded AgentV directory while retaining all gate inputs and semantic aggregates in dashboard JSON" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "deduplicate triangular depth telemetry and retain raw semantic rows at R=8 so dashboard evidence stays deployable without changing any gate input or verdict" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: publish the clean reproducible SLM-243 report and AgentV bundle generated against implementation commit c926cb05" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: make check mode replay the committed version stamp so durable evidence remains reproducible after its docs-only commit" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: serialize AgentV paths relative to the durable evidence directory so reports are machine-portable" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "no-bump: add deterministic mechanism fixtures proving historical empty-F pass-through, exact true-empty zero, near-identity scales and gates, and private norm identity" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial six-arm, five-depth, three-paired-seed recurrence update, empty-F, and private-norm architecture gate" - } - ] - }, - "harness.experiments.slm233_recursive_campaign": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm233_recursive_campaign.py", - "scripts/run_slm233_recursive_campaign.py", - "tests/test_harnesses/experiments/test_slm233_recursive_campaign.py", - "tests/test_scripts/test_run_slm233_recursive_campaign.py", - "docs/design/iter-slm233-recursive-campaign-20260724.json", - "docs/design/iter-slm233-recursive-campaign-20260724.md", - "docs/design/iter-slm233-recursive-campaign-agentv-20260724/" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-24", - "note": "no-bump: remove Markdown hard-break trailing spaces so repository diff validation remains clean" - }, - { - "version": "v2", - "date": "2026-07-24", - "note": "no-bump: make Markdown rendering stable across in-memory tuples and JSON-reloaded lists without changing scientific payloads" - }, - { - "version": "v2", - "date": "2026-07-24", - "note": "no-bump: publish the clean compact report, raw matrix, AgentV bundle, and measured-results narrative generated against implementation commit 6852e2cb" - }, - { - "version": "v2", - "date": "2026-07-24", - "note": "scope common initialization hash equality within each paired seed instead of incorrectly requiring different seed initializations to be identical" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "initial five-arm, three-paired-seed, gate-bounded matched recursive-depth campaign with fairness, exposure, test-depth, cost, resume, AgentV, and RecursiveCoreGateV2 evidence" - } - ] - }, - "harness.experiments.slm236_recurrent_latent_disposition": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm236_recurrent_latent_disposition.py", - "scripts/publish_recurrent_latent_disposition.py", - "tests/test_harnesses/experiments/test_slm236_recurrent_latent_disposition.py", - "tests/test_scripts/test_publish_recurrent_latent_disposition.py", - "docs/design/iter-slm236-recurrent-latent-disposition-20260724.json", - "docs/design/recurrent-semantic-computation-looped-latent-disposition.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial fail-closed RSC4 disposition assembled from merged gate evidence" - } - ] - }, - "harness.experiments.slm288_meaningful_validation": { - "version": "v1", - "kind": "harness", - "paths": [ - "scripts/run_slm288_meaningful_validation.py", - "tests/test_scripts/test_run_slm288_meaningful_validation.py", - "src/slm_training/resources/evals/slm288_annotation_packet_v1.json", - "src/slm_training/resources/evals/slm288_meaningful_rubric_v1.md", - "docs/design/iter-slm288-meaningful-validation-20260724.json", - "docs/design/iter-slm288-meaningful-validation-20260724.md", - "docs/design/slm288-annotation-labels-20260724.json" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-288 reason-code histograms, frozen blinded annotation packet, agent-rater agreement analysis, and metric disposition" - } - ] - }, - "harness.experiments.slm299_edit_reachability": { - "version": "v4", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm299_edit_reachability.py", - "scripts/run_slm299_reachability_audit.py", - "tests/test_harnesses/experiments/test_slm299_edit_reachability.py", - "docs/design/iter-slm299-edit-reachability-20260724.json", - "docs/design/iter-slm299-edit-reachability-20260724.md", - "src/slm_training/models/tree_edit_diffusion.py", - "docs/design/iter-slm305-edit-language-20260724.json", - "docs/design/iter-slm305-edit-language-20260724.md", - "tests/test_models/test_tree_edit_diffusion.py" - ], - "history": [ - { - "version": "v4", - "date": "2026-07-24", - "note": "SLM-310: tree_edit_diffusion gains reason-coded apply rejections, per-proposal decode telemetry (applicability/rejection/budget), corruption_action_distribution + stop_slot_accounting config knobs (defaults = historical behavior, checkpoint parity); test_tree_edit_diffusion unchanged semantics" - }, - { - "version": "v3", - "date": "2026-07-24", - "note": "SLM-308: tree_edit_diffusion gains value_label_mode (bounded_distance default for new trainings, mutation_count parity for pre-field checkpoints), oracle-backed normalized cost-to-go value targets with UNKNOWN exclusion, and pairwise parent/improving-child progress loss; edit space and reachability machinery unchanged" - }, - { - "version": "v2", - "date": "2026-07-24", - "note": "SLM-305: extend the valid-state edit language (ADD_CONTAINER/REMOVE_CONTAINER, INSERT/REPLACE_SUBTREE, V0.5 statement edits, BIND_PLACEHOLDER) with inverses, parser-backed validation, checkpoint format 2 + migration, and extended reachability mode" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; real X22 tree-edit reachability analyzer, suite audit CLI, and reachability-unproven gate input" - } - ] - }, - "harness.experiments.slm301_prompt_observability": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm301_prompt_observability.py", - "scripts/run_slm301_prompt_observability.py", - "tests/test_harnesses/experiments/test_slm301_prompt_observability.py", - "docs/design/iter-slm301-prompt-observability-20260724.json", - "docs/design/iter-slm301-prompt-observability-20260724.md", - "docs/design/slm301-slot-observability-audit-20260724.json" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-301 LAR1-04 prompt-observability arms (as-is vs +inventory) with phase-0 coverage classification, AR/X22 baselines, paired predeclared-delta report" - } - ] - }, - "harness.experiments.slm303_decode_budget_audit": { - "version": "v1", - "kind": "harness", - "paths": [ - "scripts/run_slm303_decode_budget_audit.py", - "tests/test_harnesses/model_build/test_decode_outcome.py", - "tests/test_scripts/test_run_slm303_decode_budget_audit.py", - "docs/design/iter-slm303-decode-budget-audit-20260724.json", - "docs/design/iter-slm303-decode-budget-audit-20260724.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-303 decode-budget audit with hash pins, preregistered budget sweep, and taxonomy disposition" - } - ] - }, - "harness.experiments.slm308_distance_value": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm308_distance_oracle.py", - "scripts/run_slm308_distance_value.py", - "tests/test_harnesses/experiments/test_slm308_distance_value.py", - "docs/design/iter-slm308-distance-value-20260724.json", - "docs/design/iter-slm308-distance-value-20260724.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-308 LAR2-02 bounded reverse-BFS distance oracle (EXACT/BOUNDED/UNKNOWN) over canonical AST fingerprints, distance value supervision + pairwise progress ranking in the X22 tree-edit model, matched fixture experiment with preregistered thresholds" - } - ] - }, - "harness.model_build.eval": { - "version": "v54", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/model_build/debt_router.py", - "src/slm_training/harnesses/model_build/eval_runner.py", - "src/slm_training/harnesses/model_build/eval_policy.py", - "src/slm_training/models/decode_stats.py", - "src/slm_training/harnesses/quality/retrieval.py", - "scripts/evaluate_model.py" - ], - "history": [ - { - "version": "v54", - "date": "2026-07-24", - "note": "Add bounded opt-in LangSmith aggregate telemetry to the shared evaluation trace while keeping local evidence authoritative" - }, - { - "version": "v53", - "date": "2026-07-24", - "note": "Add SLM-303 per-record decode outcome taxonomy and suite outcome counts without changing existing gate inputs" - }, - { - "version": "v52", - "date": "2026-07-24", - "note": "Add typed meaningful-program v1 reason reports while preserving the existing verdict and legacy reason strings" - }, - { - "version": "v51", - "date": "2026-07-24", - "note": "Publish AgentEvals-authoritative criteria with every policy evaluation and route CLI ship outcomes through that evidence" - }, - { - "version": "v50", - "date": "2026-07-24", - "note": "SLM-342: overlay explicit OpenFeature research-lever assignments on evaluation config and persist them in emitted results" - }, - { - "version": "v49", - "date": "2026-07-24", - "note": "Persist suite harness provenance and explicit raw, constrained, and repaired lineage IDs on every evaluation detail" - }, - { - "version": "v48", - "date": "2026-07-23", - "note": "Resolve and persist OpenFeature evaluation-lever snapshots in scoreboards" - }, - { - "version": "v47", - "date": "2026-07-23", - "note": "Keep power utilities off the import path, complete exact-match evidence, and represent non-binomial decode telemetry without invalid Wilson counts" - }, - { - "version": "v46", - "date": "2026-07-23", - "note": "Attach numerator, denominator, seed count, Wilson interval, and evidence class to canonical binomial rates" - }, - { - "version": "v45", - "date": "2026-07-22", - "note": "Count certified finalization fallbacks in decode telemetry and honest fallback gates" - }, - { - "version": "v44", - "date": "2026-07-22", - "note": "Add provenance-stamped eval offsets to sample non-prefix records locally without cache collisions" - }, - { - "version": "v43", - "date": "2026-07-22", - "note": "Include the canonical lever-contract version in eval cache identity and every suite or scoreboard version stamp" - }, - { - "version": "v42", - "date": "2026-07-22", - "note": "Persist the complete effective post-checkpoint model configuration in every evaluation policy and cache identity" - }, - { - "version": "v41", - "date": "2026-07-22", - "note": "Persist every ModelBuildConfig decode weight from the canonical field registry instead of a partial hand-maintained list" - }, - { - "version": "v40", - "date": "2026-07-21", - "note": "Keep atomic strict decode policy from mutating checkpoint schema, slot-contract, or design input conditioning" - }, - { - "version": "v39", - "date": "2026-07-21", - "note": "Normalize compiler-tree evaluation to one atomic strict policy before artifacts are created" - }, - { - "version": "v38", - "date": "2026-07-21", - "note": "Make free-form output strings an explicit meaningful_program_v1 failure" - }, - { - "version": "v37", - "date": "2026-07-21", - "note": "declare legacy visible slot-suffix semantic roles as typed runtime-symbol authority at the canonical evaluation request boundary" - }, - { - "version": "v36", - "date": "2026-07-21", - "note": "Merge E631 meaningful-program evaluation tightening with E675 and E687 evaluation policy and trace persistence" - }, - { - "version": "v35", - "date": "2026-07-21", - "note": "E687: attach stable evaluation record IDs to per-call constrained-selection traces before aggregation" - }, - { - "version": "v34", - "date": "2026-07-21", - "note": "E675: expose default-off schema open-property decode margin in evaluation policy and evidence" - }, - { - "version": "v34", - "date": "2026-07-21", - "note": "E631: _is_meaningful_program (meaningful_program_v1) gains an AST-structural empty-children check (_first_empty_children_component, walking Program.root) alongside its existing Stack([])/Card([]) literal substring checks -- purely additive, never removes an existing rejection. Fixes the gap E630 flagged and deferred: a component whose children array is genuinely empty but whose remaining non-content positional arguments got padded with unrelated stuffed values (e.g. `Card([], \":a\", \":b\")`) never matched the literal \"Card([])\" substring, so it passed as meaningful when it should not have. The AST check reads children from the parsed props dict, which is structurally separate from a node's other props, so padding those other props cannot hide an empty children array. Generalizes for free to any schema component the parser itself marks with a children prop (Modal, Carousel), which the old literal check never covered at all (new reason code `empty_children:`; Stack/Card keep their original `empty_root_stack`/`empty_card` reason strings for backward compatibility). Verified against the real scratch checkpoint reused from E626/E628/E630 (sha256 c5b7c807...dd561221): re-scoring the already-recorded rico_held suite at required_slot_margin_decode_weight=0 (pure control, no lever active) flips rico_eval_test_42 and rico_eval_test_77 from previously (incorrectly) True to correctly False (both have a `Card([], \":cardN.title\")`-shaped padded-empty Card already present in the checkpoint's baseline decode, unrelated to the lever) -- an honest, expected tightening, not a regression. New/updated tests in tests/test_evals/test_meaningful_program.py. See docs/design iter-e631." - }, - { - "version": "v33", - "date": "2026-07-20", - "note": "E626: expose and persist the default-off required-slot margin decode weight" - }, - { - "version": "v33", - "date": "2026-07-21", - "note": "SLM-212: add default-off constraint-debt routing config, DebtRoutingPolicy, CalibratedDebtRouter, CLI flags, and decode identity plumbing" - }, - { - "version": "v32", - "date": "2026-07-20", - "note": "no-bump: describe E633's model-side pre-content string routing in existing schema-opaque CLI help without changing evaluation plumbing" - }, - { - "version": "v32", - "date": "2026-07-20", - "note": "no-bump: restore the original schema-opaque CLI description when rejecting E632 without changing evaluation plumbing" - }, - { - "version": "v32", - "date": "2026-07-20", - "note": "no-bump: align schema-opaque CLI help with the expanded model-side non-content string policy without changing evaluation plumbing" - }, - { - "version": "v32", - "date": "2026-07-20", - "note": "no-bump: describe the expanded model-side slot coverage closure policy in CLI help without changing evaluation plumbing" - }, - { - "version": "v32", - "date": "2026-07-20", - "note": "expose and persist the default-off schema-derived typed-array item margin" - }, - { - "version": "v31", - "date": "2026-07-20", - "note": "expose and persist the default-off authored typed-array nonempty margin" - }, - { - "version": "v30", - "date": "2026-07-20", - "note": "expose and persist the default-off repeated-plan distinct-slot margin" - }, - { - "version": "v29", - "date": "2026-07-20", - "note": "expose and persist the default-off repeated-plan-family nested-array close margin" - }, - { - "version": "v28", - "date": "2026-07-20", - "note": "make the reported evaluation canvas cap govern request-aware generation length" - }, - { - "version": "v27", - "date": "2026-07-20", - "note": "expose and persist the default-off verifier-approved semantic-plan root margin" - }, - { - "version": "v26", - "date": "2026-07-20", - "note": "expose and persist the default-off semantic-plan legal-component margin score" - }, - { - "version": "v25", - "date": "2026-07-20", - "note": "expose the default-off first-component semantic-plan seed score in canonical evaluation policy" - }, - { - "version": "v24", - "date": "2026-07-20", - "note": "persist the default-off visible-slot coverage close score in canonical evaluation policy" - }, - { - "version": "v23", - "date": "2026-07-20", - "note": "persist the default-off public-schema semantic-role candidate policy" - }, - { - "version": "v22", - "date": "2026-07-20", - "note": "expose the default-off inline semantic-plan family score and persist it in canonical evaluation policy" - }, - { - "version": "v21", - "date": "2026-07-20", - "note": "expose the default-off enum-argument close score and persist it in canonical evaluation policy" - }, - { - "version": "v20", - "date": "2026-07-20", - "note": "expose the default-off schema-role slot score and persist it in canonical evaluation policy" - }, - { - "version": "v19", - "date": "2026-07-20", - "note": "expose the default-off optional opaque-argument close score and persist it in canonical evaluation policy" - }, - { - "version": "v18", - "date": "2026-07-20", - "note": "expose the default-off schema-opaque decode weight and persist it in canonical evaluation policy" - }, - { - "version": "v17", - "date": "2026-07-20", - "note": "expose the default-off schema-value decode weight and persist it in canonical evaluation policy" - }, - { - "version": "v16", - "date": "2026-07-20", - "note": "expose plan-coverage-gated legal Stack root construction and termination scoring in canonical evaluation" - }, - { - "version": "v15", - "date": "2026-07-20", - "note": "expose prompt-plan-compatible terminal-root reference soft scoring and its bounded intervention telemetry" - }, - { - "version": "v14", - "date": "2026-07-20", - "note": "expose prompt-derived SemanticPlanV1 soft scoring for legal choice-codec root and bound component decisions and stamp the TwoTower revision in its eval results" - }, - { - "version": "v13", - "date": "2026-07-19", - "note": "expose learned terminal-root reference identity bias and intervention telemetry in canonical evaluation" - }, - { - "version": "v12", - "date": "2026-07-19", - "note": "bound learned root-reference arity partitioning by the available generated sections so impossible tokenizer-tail classes cannot force continuation" - }, - { - "version": "v11", - "date": "2026-07-19", - "note": "add learned choice-codec terminal-root reference arity decoding and bounded intervention telemetry" - }, - { - "version": "v10", - "date": "2026-07-19", - "note": "restrict structural generated-reference completeness bias to the terminal root list" - }, - { - "version": "v9", - "date": "2026-07-19", - "note": "persist bounded root-vs-nested choice-frame and reference-bias counterfactual telemetry" - }, - { - "version": "v8", - "date": "2026-07-19", - "note": "extend generated-reference completeness bias to reachable structural list aggregation while remaining fail-closed elsewhere" - }, - { - "version": "v7", - "date": "2026-07-19", - "note": "persist bounded choice-token and legal-reference decision evidence for per-record eval diagnosis" - }, - { - "version": "v6", - "date": "2026-07-19", - "note": "add a fail-closed choice-codec bias toward unused legal generated bound references in the root" - }, - { - "version": "v5", - "date": "2026-07-19", - "note": "add a direct constrained-decode bias from honest visible semantic-role candidates" - }, - { - "version": "v4", - "date": "2026-07-19", - "note": "add honest inference-time semantic-role contract projection from prompt-mentioned components and visible slots" - }, - { - "version": "v3", - "date": "2026-07-19", - "note": "merge SDE3-01 suite-level eval cache integration on top of decode-weight override CLI" - }, - { - "version": "v2", - "date": "2026-07-19", - "note": "add SDE3-01 suite-level content-addressed eval cache integration (EvalCache, --eval-cache-mode, --eval-cache-root)" - }, - { - "version": "v2", - "date": "2026-07-19", - "note": "expose the existing slot-to-component decode-weight override in the canonical evaluation CLI" - }, - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration" - } - ] - }, - "harness.model_build.data": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/model_build/data.py" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "reject malformed or metadata-divergent Harness DSL prompts before train or eval records reach model input construction" - }, - { - "version": "v1", - "date": "2026-07-22", - "note": "Enforce the symbol-only completion contract in shared train and eval record loading before model construction or artifacts" - } - ] - }, - "harness.model_build.train": { - "version": "v24", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/model_build/train_loop.py", - "src/slm_training/harnesses/model_build/full_state.py" - ], - "history": [ - { - "version": "v24", - "date": "2026-07-23", - "note": "Reject disposition-forbidden spectral controls before direct ship evaluation or promoted training" - }, - { - "version": "v23", - "date": "2026-07-23", - "note": "Fail staged training before corpus or model initialization when capability certificates or lever permissions are invalid" - }, - { - "version": "v22", - "date": "2026-07-23", - "note": "Persist resolved OpenFeature training-lever snapshots" - }, - { - "version": "v21", - "date": "2026-07-23", - "note": "Fail before training when explicit checkpoint promotion lacks valid campaign governance evidence" - }, - { - "version": "v20", - "date": "2026-07-22", - "note": "Restore the root-reference identity strict-subset sampler to its reachable choice-tokenizer capability" - }, - { - "version": "v19", - "date": "2026-07-22", - "note": "Select strict-subset root-reference identity records through either choice or lexer grammar-native targets" - }, - { - "version": "v18", - "date": "2026-07-21", - "note": "no-bump: source the already-v18 two-minute training cap from the canonical lever registry; runtime behavior is unchanged" - }, - { - "version": "v18", - "date": "2026-07-21", - "note": "reduce the cumulative training deadline default and maximum from three minutes to two" - }, - { - "version": "v17", - "date": "2026-07-21", - "note": "SLM-222: add Muon/AdamW hybrid optimizer support with optimizer-family fingerprint and fail-closed resume guard; default AdamW path preserved" - }, - { - "version": "v16", - "date": "2026-07-20", - "note": "wire SDE2-03 exposure-targeted rare-action sampler kwargs and recipe metadata; default with_replacement sampler unchanged" - }, - { - "version": "v15", - "date": "2026-07-20", - "note": "add deterministic rare slot-owner record oversampling with auditable corpus counts and expanded exposure" - }, - { - "version": "v14", - "date": "2026-07-20", - "note": "persist slot next-context and pair-interaction architecture flags in train recipe summaries" - }, - { - "version": "v13", - "date": "2026-07-20", - "note": "rebuild deterministic corpus-derived slot priors after weight initialization instead of silently restoring stale parent checkpoint statistics" - }, - { - "version": "v12", - "date": "2026-07-19", - "note": "add deterministic strict-subset root-reference identity oversampling with auditable eligible and expanded sampling counts" - }, - { - "version": "v11", - "date": "2026-07-19", - "note": "record configurable excluded-reference loss weighting and negative-class calibration telemetry for terminal-root identity training" - }, - { - "version": "v10", - "date": "2026-07-19", - "note": "record learned terminal-root reference identity loss and decode settings in training recipes" - }, - { - "version": "v9", - "date": "2026-07-19", - "note": "mask impossible root-reference arity classes by each target stream's available generated sections during auxiliary training" - }, - { - "version": "v8", - "date": "2026-07-19", - "note": "train an isolated choice-codec terminal-root reference arity head and record its recipe settings" - }, - { - "version": "v7", - "date": "2026-07-19", - "note": "no-bump: isolate the train.progress heartbeat behind a warn-only guard so telemetry write failures can never abort training" - }, - { - "version": "v7", - "date": "2026-07-19", - "note": "no-bump: throttled OTLP train.progress heartbeat logs for live peer streaming; training math, metrics.jsonl rows, and summaries unchanged" - }, - { - "version": "v7", - "date": "2026-07-19", - "note": "expose honest prompt-derived slot-contract training and record contract provenance in train summaries" - }, - { - "version": "v6", - "date": "2026-07-19", - "note": "add bounded source-stratified masked-token loss proxy summaries for parent-replay attribution" - }, - { - "version": "v5", - "date": "2026-07-19", - "note": "add deterministic hash-pinned parent-corpus replay with fail-closed sampler composition and effective-exposure telemetry" - }, - { - "version": "v4", - "date": "2026-07-19", - "note": "add explicit initialized-weight retention with strict initialization guards and RMS drift telemetry" - }, - { - "version": "v3", - "date": "2026-07-18", - "note": "preserve corpus-derived serving priors during checkpoint initialization and expose restored fields plus slot-head recipe settings in train summaries" - }, - { - "version": "v2", - "date": "2026-07-18", - "note": "add explicit weight-only checkpoint initialization for matched new-corpus training without weakening bit-exact resume guards" - }, - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration" - } - ] - }, - "harness.oracle_scoring_replay": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/evals/oracle_scoring_replay.py", - "scripts/audit_gold_scoring.py", - "tests/test_evals/test_oracle_scoring_replay.py", - "docs/design/oracle-scoring-replay.md", - "docs/design/iter-slm260-oracle-scoring-replay-20260721.json", - "docs/design/iter-slm260-oracle-scoring-replay-20260724.json" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "no-bump: record a local 66-row oracle replay under unchanged scorer and harness behavior" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-260 VSD0-01 oracle scoring replay harness and audit CLI" - } - ] - }, - "harness.eval.semantic_failure_census": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/evals/semantic_failure.py", - "scripts/audit_semantic_failures.py", - "tests/test_evals/test_semantic_failure.py", - "tests/test_scripts/test_audit_semantic_failures.py", - "docs/design/semantic-failure-census.md", - "docs/design/iter-slm263-semantic-failure-census-20260724.json" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-263 deterministic first-semantic-failure taxonomy and immutable-generation census" - } - ] - }, - "harness.experiments.slm265_domain_shift_audit": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm265_domain_shift_audit.py", - "scripts/audit_domain_shift.py", - "tests/test_harnesses/experiments/test_slm265_domain_shift_audit.py", - "docs/design/rico-domain-shift-audit.md", - "docs/design/iter-slm265-domain-shift-audit-20260724.json" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-265 outcome-blind domain-shift strata and coverage-gap audit" - } - ] - }, - "harness.experiments.slm266_teacher_programs": { - "version": "v6", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/teacher_programs.py", - "scripts/build_teacher_program_generation_manifest.py", - "scripts/admit_teacher_programs.py", - "scripts/generate_teacher_programs.py", - "scripts/materialize_teacher_programs.py", - "tests/test_harnesses/experiments/test_teacher_programs.py", - "docs/design/teacher-program-data.md" - ], - "history": [ - { - "version": "v6", - "date": "2026-07-24", - "note": "add revision-pinned local-only Transformers teacher transport and CLI execution mode" - }, - { - "version": "v5", - "date": "2026-07-24", - "note": "bind deep teacher admission G11 to persisted cross-family automatic judge evidence" - }, - { - "version": "v4", - "date": "2026-07-24", - "note": "execute locked teacher requests through budgeted provider transport with content-addressed raw attempt archive" - }, - { - "version": "v3", - "date": "2026-07-24", - "note": "materialize only deep-admitted teacher rows as canonical train snapshots with immutable raw archive pointer" - }, - { - "version": "v2", - "date": "2026-07-24", - "note": "make G12 optional evidence and derive Silver admission only from automated deep verification" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-266 fail-closed teacher-program request and offline admission contracts" - } - ] - }, - "harness.eval.semantic_trajectory": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/evals/semantic_trajectory.py", - "scripts/collect_semantic_trajectory.py", - "tests/test_evals/test_semantic_trajectory.py", - "src/slm_training/resources/evals/slm264_trajectory_fixture.jsonl", - "docs/design/semantic-collapse-telemetry.md", - "docs/design/iter-slm264-semantic-collapse-telemetry-20260724.json" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-264 default-off semantic trajectory schema and evidence inventory" - } - ] - }, - "harness.preference.constraint_debt": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/preference/constraint_debt.py", - "src/slm_training/harnesses/preference/local_train.py", - "scripts/diagnose_preference_gradient_alignment.py", - "tests/test_harnesses/preference/test_constraint_debt.py", - "tests/test_harnesses/preference/test_local_train_constraint_debt.py", - "docs/design/iter-slm208-constraint-debt-20260720.json", - "docs/design/iter-slm208-constraint-debt-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: SLM-209 reuses ConstraintDebtV1 rows and compute_constraint_debt_v1 types only; no behavior changed" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-208 SDE5-01 constraint-debt V1 telemetry and pre/post-mask grammar-distortion KL wiring" - } - ] - }, - "harness.preference.remine_campaign": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/preference/remine_campaign.py", - "scripts/run_remine_campaign.py", - "scripts/autoresearch.py", - "tests/test_harnesses/preference/test_remine_campaign.py", - "tests/test_scripts/test_autoresearch_remine.py", - "docs/design/ldi3-04-remine-intervene-campaign-20260718.md", - "docs/design/ldi3-04-remine-intervene-campaign-report-20260718.json" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: scripts/autoresearch.py adds independent campaign-lock enforcement; remine campaign behavior is unchanged" - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "no-bump: source the already-v2 remine and autoresearch cap from the canonical lever registry; runtime behavior is unchanged" - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "reduce remine and autoresearch campaign wall-budget default and maximum to two minutes" - }, - { - "version": "v1", - "date": "2026-07-21", - "note": "no-bump: scripts/autoresearch.py gains an export-openfeature subcommand (flagd flag-definition export of a hypothesis matrix); remine campaign behavior unchanged" - }, - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration; SLM-132 LDI3-04 immutable on-policy remine/intervene/regenerate campaign wiring/fixture harness" - } - ] - }, - "harness.quality.slop_forensics": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/quality/slop_forensics.py", - "scripts/run_slop_forensics.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration" - } - ] - }, - "harness.reasoning.bench": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/reasoning/" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration" - } - ] - }, - "harness.representations": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/representations/", - "scripts/run_sae_diagnostic_fixture.py", - "tests/test_harnesses/representations/", - "tests/test_scripts/test_run_sae_diagnostic_fixture.py", - "docs/design/iter-ldi4-02-sae-decision-state-diagnostic-20260720.json", - "docs/design/iter-ldi4-02-sae-decision-state-diagnostic-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact timestamp after fixture CLI rerun; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration of SLM-136 LDI4-02 SAE decision-state diagnostic harness (schema, SAE, interventions, fixture wiring)" - } - ] - }, - "harness.capability_artifacts": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/capability_artifacts.py", - "tests/test_harnesses/test_capability_artifacts.py", - "docs/design/dsh0-04-capability-artifacts-20260723.json", - "docs/design/dsh0-04-capability-artifacts-20260723.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial content-addressed answer, question, QA-set, derivation, validation, and capability-certificate schemas with publication lineage checks" - } - ] - }, - "harness.capability_gates": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/capability_gates.py", - "scripts/manage_capability_certificate.py", - "tests/test_harnesses/test_capability_gates.py", - "docs/design/dsh0-07-capability-gates-20260723.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "Add immutable capability gate specs, results, certificates, retention and prerequisite enforcement, and explicit promotion commands" - } - ] - }, - "harness.artifact_graph": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/train_data/artifact_graph.py", - "src/slm_training/harnesses/train_data/split_policy.py", - "scripts/explain_artifact_overlaps.py", - "tests/test_harnesses/train_data/test_artifact_graph.py", - "docs/design/dsh0-05-artifact-graph-20260723.json", - "docs/design/dsh0-05-artifact-graph-20260723.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial append-only dataset artifact graph, root-family split inheritance, cross-split leakage quarantine, traversal, and explain CLI" - } - ] - }, - "harness.solver_bench": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/solver_bench.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration" - } - ] - }, - "harness.staged": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/staged.py", - "tests/test_harnesses/test_staged.py", - "scripts/verify_foundation_disposition.py", - "docs/design/dsh0-01-staged-harness-baseline-20260723.json", - "docs/design/dsh0-01-staged-harness-baseline-20260723.md", - "docs/design/dsh0-02-symbolic-surface-policy-20260723.json", - "docs/design/dsh0-02-symbolic-surface-policy-20260723.md", - "docs/design/dsh0-08-foundation-disposition-20260723.json", - "docs/design/dsh0-08-foundation-disposition-20260723.md" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: refresh pinned pack, CAP0 plan, materialization, and disposition hashes after adding the independent Harness DSL boundary" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: refresh pinned DslPack hashes after adding independent minimal-witness authority" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: refresh pinned DslPack source hashes after adding the independent grammar capability authority slot" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "Add the typed fail-closed G0 foundation disposition, current-artifact verification, and exact CAP0 identity freeze" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial staged DSL capability taxonomy and fail-closed StagedHarnessBaselineV1 evidence identity contract" - } - ] - }, - "harness.synthesis_plan": { - "version": "v2", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/synthesis_plan.py", - "src/slm_training/resources/synthesis_plans/", - "tests/test_harnesses/test_synthesis_plan.py", - "docs/design/dsh0-03-synthesis-plan-20260723.json", - "docs/design/dsh0-03-synthesis-plan-20260723.md" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "no-bump: refresh the executable CAP0 fixture to the v18 symbolic task materialization contract" - }, - { - "version": "v2", - "date": "2026-07-23", - "note": "refresh the executable CAP0 fixture to the v17 train-data materialization contract and preserve its prior identity as historical evidence" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial checked-in SynthesisPlanV1, pack-authoritative component preflight, and executable capability transition state machine" - } - ] - }, - "harness.train_data": { - "version": "v19", - "kind": "harness", - "paths": [ - "scripts/build_train_data.py", - "src/slm_training/harnesses/train_data/", - "src/slm_training/data/language_contract/", - "src/slm_training/dsl/renderability.py", - "src/slm_training/data/mixture.py", - "src/slm_training/dsl/packs/openui.py", - "src/slm_training/resources/train_seeds.jsonl", - "src/slm_training/resources/frontier/" - ], - "history": [ - { - "version": "v19", - "date": "2026-07-23", - "note": "add runtime-renderability repairs and chosen-over-rejected preference pairs for structural-only OpenUI roots" - }, - { - "version": "v18", - "date": "2026-07-23", - "note": "emit identity and canonical CAP0 scope rows as validated symbolic Harness DSL while preserving legacy repair and typed-map semantics" - }, - { - "version": "v17", - "date": "2026-07-23", - "note": "materialize synthesis-plan artifact graphs through canonical ExampleRecord, validation, preference, quality, leakage, and publication paths" - }, - { - "version": "v16", - "date": "2026-07-23", - "note": "add staged dataset artifact-graph sidecars and root-family cross-split leakage firewall" - }, - { - "version": "v15", - "date": "2026-07-23", - "note": "validate optional checked-in SynthesisPlanV1 before loading any train-data producer and record its exact identity in the manifest" - }, - { - "version": "v14", - "date": "2026-07-23", - "note": "remove the train_text_only_01 source fixture after strict sanitization proved it structurally duplicates the reserved adv_empty_prompt_01 test fallback" - }, - { - "version": "v13", - "date": "2026-07-23", - "note": "emit symbol-only collapsed multi-instruction operator records with exact trace and hard-negative evidence" - }, - { - "version": "v12", - "date": "2026-07-23", - "note": "no-bump: pin operator contract and legal-set component versions in the sibling report stamp; generated corpus behavior is unchanged" - }, - { - "version": "v12", - "date": "2026-07-23", - "note": "generate replay-verified symbolic single-turn, next-turn, and sibling-fork operator datasets from admitted exact AST roots" - }, - { - "version": "v11", - "date": "2026-07-22", - "note": "Templatize every OpenUI pack-generator target before record emission and remove stale free-form settings names from canonical train seeds" - }, - { - "version": "v10", - "date": "2026-07-21", - "note": "Fail closed unless sanitization has templatized every non-grammar string, including arrays and operational string fields" - }, - { - "version": "v9", - "date": "2026-07-20", - "note": "add SDE2-03 exposure-targeted rare-action online sampler with bounded importance weights and diversity caps; default with_replacement unchanged" - }, - { - "version": "v8", - "date": "2026-07-19", - "note": "add deterministic target sanitization (D2 canonicalization + schema-checked default-elision/dead-binding removal/guarded Stack flattening + content-literal templatization) behind sanitize_mode off/audit/enforce; strict profile defaults to enforce" - }, - { - "version": "v7", - "date": "2026-07-19", - "note": "no-bump: reuse the semantic-role formatter from the shared quality owner without changing projected training records" - }, - { - "version": "v7", - "date": "2026-07-19", - "note": "add an opt-in visible semantic-role contract derived only from visible slots, visible component types, and generated schema compatibility" - }, - { - "version": "v6", - "date": "2026-07-19", - "note": "support a type-only visible component prompt contract for matched count-ablation data" - }, - { - "version": "v5", - "date": "2026-07-19", - "note": "add diversity fingerprinting module (diversity.py), corpus audit script, and regression tests for canonical-root, topology, type/action, prompt-intent, and lineage diversity metrics" - }, - { - "version": "v4", - "date": "2026-07-19", - "note": "add an opt-in post-admission visible component type/count contract to training prompts" - }, - { - "version": "v3", - "date": "2026-07-19", - "note": "add synthetic-data integrity gate module (integrity.py), audit script, and regression tests for canonical AST, codec round-trip, binding graph, slot contract, and leakage checks" - }, - { - "version": "v2", - "date": "2026-07-18", - "note": "add provenance-tagged expression-to-document projection and explicit target-kind corpus selection" - }, - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration; strict build results carry version stamps and actionable rejection lineage" - } - ] - }, - "harness_core.lineage.interventions": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harness_core/lineage/interventions.py", - "scripts/run_intervention_unification_fixture.py", - "tests/test_lineage_interventions.py", - "tests/test_scripts/test_run_intervention_unification_fixture.py", - "docs/design/iter-ldi4-03-intervention-unification-20260720.json", - "docs/design/iter-ldi4-03-intervention-unification-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration of SLM-137 LDI4-03 unified intervention manifest/registry/promotion harness" - } - ] - }, - "matrix.b3_capacity_v2": { - "version": "v1", - "kind": "matrix", - "paths": [ - "scripts/run_b3_capacity_v2.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration; SLM-124 EFS3-03 B3 surface-vs-choice capacity-ladder v2 plan/fixture CLI" - } - ] - }, - "matrix.causal_peft_ftpo": { - "version": "v1", - "kind": "matrix", - "paths": [ - "scripts/run_causal_peft_ftpo.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration; SLM-121 causal PEFT FTPO plan/fixture CLI" - } - ] - }, - "matrix.corruption_curriculum": { - "version": "v1", - "kind": "matrix", - "paths": [ - "scripts/run_corruption_curriculum.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration; SLM-120 near-solved semantic corruption-curriculum plan/fixture CLI" - } - ] - }, - "matrix.grammar": { - "version": "v2", - "kind": "matrix", - "paths": [ - "scripts/run_grammar_matrix.py" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "Persist OpenFeature training/evaluation flag snapshots in grammar-matrix rows" - }, - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration" - } - ] - }, - "matrix.perf": { - "version": "v1", - "kind": "matrix", - "paths": [ - "scripts/run_perf_matrix.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration" - } - ] - }, - "matrix.quality": { - "version": "v7", - "kind": "matrix", - "paths": [ - "scripts/run_quality_matrix.py" - ], - "history": [ - { - "version": "v7", - "date": "2026-07-24", - "note": "Retain authoritative evaluation criteria and runner provenance in quality-matrix summaries" - }, - { - "version": "v6", - "date": "2026-07-23", - "note": "Persist OpenFeature training/evaluation flag snapshots in quality-matrix rows" - }, - { - "version": "v5", - "date": "2026-07-22", - "note": "Centralize design-context conditioning in the shared strict compiler-tree policy inherited by V9 and V10" - }, - { - "version": "v4", - "date": "2026-07-21", - "note": "no-bump: restore historical runtime-symbol-feature experiment descriptions while documenting the separate opaque production projection; matrix dispatch and recipes unchanged" - }, - { - "version": "v4", - "date": "2026-07-21", - "note": "add SLM-210 --matrix-set sde5-floor-escape dispatch for preregistered floor-escape wiring fixture" - }, - { - "version": "v3", - "date": "2026-07-20", - "note": "add SLM-185 default-off --resolution-manifest integration; apply_resolution_manifest annotates summary with semantic_resolution labels without altering raw metrics or gates" - }, - { - "version": "v2", - "date": "2026-07-20", - "note": "no-bump: add SLM-184 default-off fixture comment/example next to evaluate_suites; runner behavior unchanged" - }, - { - "version": "v2", - "date": "2026-07-19", - "note": "add SLM-108 external-ceiling matrix set dispatch (--matrix-set external-ceiling)" - }, - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration" - } - ] - }, - "matrix.slm189_bridge_planner": { - "version": "ffe2-01-v1", - "kind": "matrix", - "paths": [ - "scripts/run_bridge_planner_audit.py", - "docs/design/iter-slm189-bridge-planner-20260721.json", - "docs/design/iter-slm189-bridge-planner-20260721.md" - ], - "history": [ - { - "version": "ffe2-01-v1", - "date": "2026-07-21", - "note": "initial registration; SLM-189 FFE2-01 bridge planner exact/dependency/random arm fixture matrix" - } - ] - }, - "matrix.slm190_exact_flow": { - "version": "ffe2-02-v1", - "kind": "matrix", - "paths": [ - "scripts/run_exact_flow_fixture.py", - "docs/design/iter-slm190-exact-flow-20260721.json", - "docs/design/iter-slm190-exact-flow-20260721.md" - ], - "history": [ - { - "version": "ffe2-02-v1", - "date": "2026-07-21", - "note": "initial registration; SLM-190 FFE2-02 exact CTMC enumeration/path/sample/objective/lumpability fixture matrix" - } - ] - }, - "matrix.slm191_termination_matrix": { - "version": "ffe2-03-v1", - "kind": "matrix", - "paths": [ - "scripts/run_termination_matrix.py", - "tests/test_scripts/test_run_termination_matrix.py", - "docs/design/iter-slm191-termination-matrix-20260721.json", - "docs/design/iter-slm191-termination-matrix-20260721.md" - ], - "history": [ - { - "version": "ffe2-03-v1", - "date": "2026-07-23", - "note": "no-bump: minify the unchanged measured JSON payload to restore Vercel function-size headroom; semantic JSON content and matrix behavior are identical" - }, - { - "version": "ffe2-03-v1", - "date": "2026-07-21", - "note": "no-bump: CLI test passes --no-write-design-docs so committed design docs are not overwritten during pytest; matrix semantics unchanged" - }, - { - "version": "ffe2-03-v1", - "date": "2026-07-21", - "note": "initial registration; SLM-191 FFE2-03 termination-policy fixture matrix" - } - ] - }, - "matrix.slm212_debt_routing": { - "version": "sde5-05-v1", - "kind": "matrix", - "paths": [ - "scripts/run_slm212_debt_routing_fixture.py", - "docs/design/iter-slm212-debt-routing-20260721.json", - "docs/design/iter-slm212-debt-routing-20260721.md" - ], - "history": [ - { - "version": "sde5-05-v1", - "date": "2026-07-21", - "note": "initial registration; SLM-212 SDE5-05 constraint-debt routing MaskGIT/LTR/ASAp matched-budget fixture matrix" - } - ] - }, - "matrix.verified_solver": { - "version": "vss4-02-v1", - "kind": "matrix", - "paths": [ - "src/slm_training/harnesses/experiments/verified_solver_matrix.py", - "src/slm_training/harnesses/experiments/vss4_campaign.py" - ], - "history": [ - { - "version": "vss4-02-v1", - "date": "2026-07-18", - "note": "initial registration; mirrors verified_solver_matrix.MATRIX_VERSION" - } - ] - }, - "model.global_semantic_critic": { - "version": "v1", - "kind": "model", - "paths": [ - "src/slm_training/models/global_semantic_critic.py", - "src/slm_training/models/global_semantic_critic_selector.py", - "tests/test_models/test_global_semantic_critic.py", - "tests/test_models/test_global_semantic_critic_selector.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-150 SPV2-02 global semantic energy/value critic baseline wiring/fixture harness" - } - ] - }, - "model.legal_action_scorer": { - "version": "v1", - "kind": "model", - "paths": [ - "src/slm_training/models/legal_action_scorer.py", - "scripts/run_slm154_legal_action_scorer_fixture.py", - "tests/test_models/test_legal_action_scorer.py", - "docs/design/iter-slm154-legal-action-scorer-20260720.json", - "docs/design/iter-slm154-legal-action-scorer-20260720.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-154 SPV3-01 capacity-matched autoregressive compiler-legal action scorer baseline wiring/fixture harness" - } - ] - }, - "model.legal_edit_batch": { - "version": "v2", - "kind": "model", - "paths": [ - "src/slm_training/models/legal_edit_batch.py", - "tests/test_models/test_legal_edit_batch.py" - ], - "history": [ - { - "version": "v2", - "date": "2026-07-23", - "note": "add label-free runtime packing that preserves exact candidate membership and marks every inference candidate UNKNOWN until independently certified" - }, - { - "version": "v1", - "date": "2026-07-23", - "note": "initial registration; objective-neutral exact dynamic candidate ragged batch, masks, multi-positive targets, padding, and gathered projection seam" - } - ] - }, - "model.quantization": { - "version": "v5", - "kind": "harness", - "paths": [ - "src/slm_training/models/quantization/", - "src/slm_training/models/local_action_head.py" - ], - "history": [ - { - "version": "v5", - "date": "2026-07-20", - "note": "use deterministic action-index mapping in GlobalMaskedHead and ResidualTritPlaneHead to eliminate per-process hash randomization" - }, - { - "version": "v4", - "date": "2026-07-19", - "note": "fix(cap3-05): honest physical-cost ledger rounding and byte-accounting for equal-byte ladder" - }, - { - "version": "v3", - "date": "2026-07-19", - "note": "CAP4-04 compiler-routed block sparsity and state-family experts merged on top of CAP4-02" - }, - { - "version": "v2", - "date": "2026-07-18", - "note": "CAP4-02 adaptive residual-plane scheduling (structural floor + entropy/margin/sensitivity/learned-router/oracle)" - }, - { - "version": "v1", - "date": "2026-07-18", - "note": "initial registration (CAP4-01 residual ternary planes and local scorer head)" - } - ] - }, - "model.recursive_denoiser": { - "version": "v18", - "kind": "harness", - "paths": [ - "src/slm_training/models/recursive_denoiser.py", - "scripts/run_slm138_recursive_denoiser_fixture.py", - "tests/test_models/test_recursive_denoiser.py", - "docs/design/iter-slm138-recursive-denoiser-20260720.json", - "docs/design/iter-slm138-recursive-denoiser-20260720.md", - "src/slm_training/models/rng_contract.py", - "scripts/run_rsc_a02_depth_aux_mode_factorial.py", - "docs/design/iter-slm138-recursive-denoiser-20260721.json", - "docs/design/iter-slm138-recursive-denoiser-20260721.md", - "docs/design/iter-rsc-a03-rng-determinism-contract-20260721.json", - "docs/design/iter-rsc-a03-rng-determinism-contract-20260721.md", - "docs/design/iter-rsc-a04-parity-claim-correction-20260721.json", - "docs/design/iter-rsc-a04-parity-claim-correction-20260721.md", - "src/slm_training/models/recursive_control_arms.py", - "docs/design/iter-rsc-a05-matched-recursive-control-arms-20260721.json", - "docs/design/iter-rsc-a05-matched-recursive-control-arms-20260721.md", - "docs/design/iter-slm138-recursive-denoiser-20260723.json", - "docs/design/iter-slm138-recursive-denoiser-20260723.md", - "docs/design/iter-slm279-depth-supervision-correction-20260723.json", - "docs/design/iter-slm279-depth-supervision-correction-20260723.md", - "docs/design/iter-slm282-recurrence-health-20260723.json", - "docs/design/iter-slm282-recurrence-health-20260723.md" - ], - "history": [ - { - "version": "v18", - "date": "2026-07-23", - "note": "add default-preserving SLM-243 current, delta-only, LayerScale, gated, true-empty-F, and private-norm recurrence controls" - }, - { - "version": "v17", - "date": "2026-07-23", - "note": "expose the exact additive z0 components and default-off evaluation-only recurrence path cuts for SLM-232 without changing the default transition" - }, - { - "version": "v16", - "date": "2026-07-23", - "note": "extract the canonical recurrence into mutation-free initial_transition_state and transition_step APIs so state Jacobians use the exact normal forward math and fixed runtime-symbol conditioning" - }, - { - "version": "v15", - "date": "2026-07-23", - "note": "no-bump: register the clean squash-merged v15 SLM-282 negative recurrence-health JSON and markdown, superseding the withdrawn unmerged v14 positive artifact after raw-example gating exposed the seed-1 regression" - }, - { - "version": "v15", - "date": "2026-07-23", - "note": "no-bump: make the v15 raw-example gate fail inconclusive on missing or duplicate curve depths, example ids, curves, or matched-control records; experiment and disposition semantics are unchanged" - }, - { - "version": "v15", - "date": "2026-07-23", - "note": "SLM-282 preregistration correction: evaluate the exact CE(final) <= CE(previous) <= CE(r=1) condition independently for every raw example and record example ids on adjacent-depth failures, preventing token-weighted aggregation from averaging away a later-depth regression" - }, - { - "version": "v14", - "date": "2026-07-23", - "note": "no-bump: preserve external --output-dir support by using output-dir URIs when AgentV artifacts are outside the repository; this completes the v14 portable-path correction without changing experiment semantics" - }, - { - "version": "v14", - "date": "2026-07-23", - "note": "SLM-282 evidence-writer correction: serialize AgentV spec and artifact paths relative to the repository before mirroring recurrence-health reports into docs/design, while retaining absolute paths for the runtime SDK invocation" - }, - { - "version": "v13", - "date": "2026-07-23", - "note": "SLM-282 (LAR0-02) recurrence-health audit: add the fixed as_is versus fixture-only residual_delta matched-control runner at logical depths 1/2/4 and two disjoint corruption schedules, canonical final-depth-only training with a separate evaluation draw, actual target/noisy/mask digest matching, one-forward anytime curves, seeded detached finite-difference directional gains, exact preregistered disposition, raw failure persistence, and AgentV fixture-not-ship publication" - }, - { - "version": "v12", - "date": "2026-07-23", - "note": "SLM-282 (LAR0-02) diagnostics foundation: add opt-in RecursiveDepthDiagnosticsV1 records with detached per-example y/z states, actual updates, norms, pre-state update ratios, and masked CE/accuracy/entropy/KL curves; add a diagnostics-gated fixture-only residual_delta transition mode while preserving the historical as_is forward path and all production constructor/forward defaults" - }, - { - "version": "v11", - "date": "2026-07-23", - "note": "no-bump: refresh only the clean fixture code_commit and timestamp after replaying the evidence stack onto squash-merged main; all measured arithmetic and objective fields are byte-identical" - }, - { - "version": "v11", - "date": "2026-07-23", - "note": "no-bump: register the clean committed SLM-138 rerun and dedicated SLM-279 correction-only JSON and markdown produced by the v11 fixture contract" - }, - { - "version": "v11", - "date": "2026-07-23", - "note": "SLM-279: make the SLM-138 fixture use the canonical final-primary plus intermediate-only auxiliary objective, emit the complete objective decomposition and recipe, and project dedicated correction-only old-buggy versus corrected arithmetic evidence" - }, - { - "version": "v10", - "date": "2026-07-23", - "note": "no-bump: isolate the SLM-138 fixture checkpoint in a process-unique temporary directory so concurrent changed-test shards cannot read a partially written archive; objective math and emitted evidence are unchanged" - }, - { - "version": "v10", - "date": "2026-07-23", - "note": "no-bump: isolate the SLM-138 fixture checkpoint in a process-unique temporary directory so concurrent changed-test shards cannot read a partially written archive; objective math and emitted evidence are unchanged" - }, - { - "version": "v10", - "date": "2026-07-23", - "note": "no-bump: make SLM-138 fixture and recursive-depth tests explicitly name their existing all-depth semantics while preserving their objective arithmetic" - }, - { - "version": "v10", - "date": "2026-07-23", - "note": "no-bump: extract the behavior-identical normalized recursive-depth loss reduction into a pure helper and make the production objective plus exact arithmetic and gradient-isolation tests share it" - }, - { - "version": "v10", - "date": "2026-07-21", - "note": "no-bump: updated tests/test_models/test_recursive_denoiser.py to expect the SLM-242 construction-time validation raise instead of a training_loss-time raise; the fail-closed invariant and error message content are unchanged, only the call site that triggers the error moved earlier" - }, - { - "version": "v10", - "date": "2026-07-21", - "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v10 revision (d6ff9ceb, code_dirty=false) to embed arm H (control_arm_table row). Content-only refresh -- H's row: 52,288 total parameters (+9,248 vs A, matching recursive_zstate_parameter_delta(d_model=32, max_len=256), identical to B's own delta), block_evaluations_per_forward=4 (same as B). No code change; arm_f_dual_view's active_parameter_count entries shifted slightly (68812->68818, 51922->51928) as an honest side effect of constructing one additional arm (H) earlier in the same run, shifting the ambient global-RNG position for the dual-view towers' random initial weights -- same class of drift every prior arm addition already caused; not a behavior change. Every other arm's row unchanged from the prior v9 no-bump regeneration." - }, - { - "version": "v10", - "date": "2026-07-21", - "note": "SLM-241 (RSC-A05) second follow-up: built control arm H (stop-gradient recurrence) -- the last of the eight arms; SLM-241 now has all 8 arms addressed. SharedRecursiveDenoiserTower gains detach_between_steps: bool = False (src/slm_training/models/recursive_denoiser.py): when True, recursive_outputs runs the identical forward recurrence as detach_between_steps=False (same shared blocks, same y/z update equations, byte-identical forward values for identical seed/weights/inputs) but calls .detach() on the carried-forward y/z at the end of every recursion step except the last, so a later step's loss cannot backpropagate through the recurrent chain into an earlier step's block application (though the shared weights still receive gradient from every step's own application of them). recursive_outputs also gains an additive, opt-in return_step_boundaries: bool = False flag exposing the real pre-detach y/z tensor at each step's exit, for gradient-mechanism tests. recursive_control_arms.py: H moved out of DEFERRED_ARM_IDS into BUILT_ARM_IDS (now empty, all 8 built); ARM_DENOISER_ARCH['H'] reuses B's exact 'shared_recursive' string (no new arch value -- same reuse convention as arm G), construct_arm_tower passes detach_between_steps=(arm_id == 'H'). rng_contract.py: documents (no code change to NAMESPACE_OFFSETS) that H deliberately gets no new arch_specific namespace, since it declares literally the same parameter names/shapes as B -- a fairness report must still never include both B and H together (same restriction already established for B+G), verified by a new test. tests/test_models/test_recursive_denoiser.py: +8 net tests (forward-identity bit-for-bit vs B; a hook-based gradient-divergence mechanism test proving the step-1/step-2 boundary tensor's backward hook fires for B and never fires for H when the loss is the last-depth-only logits; same-step shared-weight gradient still flows in H; params/block-evals match B exactly; config/checkpoint roundtrip; init-fairness exclusion + inclusion cases), 108 passing (was 100 + 9 new, 1 removed). Docs: docs/design/iter-rsc-a05-matched-recursive-control-arms-20260721.{json,md} extended with H's forward-identity and gradient-divergence evidence; SLM-241/RSC-A05 status is now complete (all 8 arms built, none deferred)." - }, - { - "version": "v9", - "date": "2026-07-21", - "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v9 revision (d6cf67c8, code_dirty=false) to embed arm E (control_arm_table row). Content-only refresh -- E's row: 52288 total parameters (+9248 vs A, matching recursive_zstate_parameter_delta(d_model=32, max_len=256)), block_evaluations_per_forward=2 (same as A). No code change; arm_f_dual_view section and every other arm's row unchanged from the prior v8 no-bump regeneration." - }, - { - "version": "v9", - "date": "2026-07-21", - "note": "SLM-241 (RSC-A05) follow-up: built control arm E (stacked + matched state capacity) -- new StackedMatchedStateDenoiserTower class (src/slm_training/models/recursive_denoiser.py): an unshared, non-recursive DenoiserTower subclass (same block-evaluation count as arm A) plus a learned state[max_len,d_model] + state_ctx_proj Linear(d_model,d_model) pair shape-matched to arm B's z_latent/ctx_proj, injected once into the initial hidden state before any transition block runs (never recurrently re-applied). Parameter delta over a same-n_layers arm A equals recursive_zstate_parameter_delta(d_model, max_len) exactly (verified against real constructed towers). recursive_control_arms.py: E moved from DEFERRED_ARM_IDS to BUILT_ARM_IDS (only H remains deferred), construct_arm_tower/build_arm_report/ARM_* registries extended. rng_contract.py: added the arch_specific:stacked_matched_state reserved namespace (offset 110_000), same reserved/not-yet-consumed convention as every other arch_specific:* entry. tests/test_models/test_recursive_denoiser.py: +9 tests (arm E structure/block-eval-count/parameter-formula/gradient-consumption/config-roundtrip/init-fairness, plus one amended table assertion and one deferred-arm-set assertion), net 100 passing (was 94 + 6 new, none removed). Docs: docs/design/iter-rsc-a05-matched-recursive-control-arms-20260721.{json,md} extended with arm E's formula/verification/gradient-consumption-test evidence." - }, - { - "version": "v8", - "date": "2026-07-21", - "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v8 revision (f800cccb, code_dirty=false) to embed arm F (control_arm_table row + arm_f_dual_view section). Content-only refresh -- note: this run also surfaced a pre-existing, code-independent tokenizer-vocab-size drift versus the historical doc (43040 vs 42752 stacked-arm total parameters at this fixture config) reproduced identically on a clean checkout of the prior commit (93ac5062) in an isolated worktree, so it predates and is unrelated to this arm-F change; relative deltas/formulas (e.g. B/G +9248 vs A, matching recursive_zstate_parameter_delta) are unaffected and still verified." - }, - { - "version": "v8", - "date": "2026-07-21", - "note": "SLM-241 (RSC-A05) follow-up: built control arm F (unshared depth-matched tower) -- recursive_control_arms.construct_arm_tower('F', ...) now returns a plain DenoiserTower with recursive_steps * recursive_transition_layers independent blocks (no longer NotImplementedError); added build_arm_f_dual_view (block-evaluation-matched vs parameter-nearest real-measured constructions, honest residual on whichever dimension isn't exact); rng_contract.py gained the arch_specific:stacked_depth_matched reserved namespace (offset 100_000); fixture script gained an arm_f_dual_view report field + markdown section. BUILT_ARM_IDS now (A,B,C,D,F,G); DEFERRED_ARM_IDS now (E,H). No change to A/B/C/D/G construction or reported values." - }, - { - "version": "v7", - "date": "2026-07-21", - "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v7 revision (8f524efc, code_dirty=false) to embed the SLM-241 (RSC-A05) control_arm_table section -- content-only refresh of already-committed evidence (verdict/losses/deep-supervision metrics unchanged), no further code/gate/harness change." - }, - { - "version": "v7", - "date": "2026-07-21", - "note": "SLM-241 (RSC-A05): added z_state_mode ('full'|'y_only'|'parameter_free') to SharedRecursiveDenoiserTower -- 'y_only' (control arm C) removes the distinct z state entirely (both F/G updates fold through y, no z_latent/ctx_proj tensor); 'parameter_free' (arm D) keeps the y/z split but replaces the learned z_latent/ctx_proj with a deterministic pooled-context broadcast, so D's total parameter count matches a stacked baseline with the same recursive_transition_layers/n_layers exactly. Added src/slm_training/models/recursive_control_arms.py -- the A-H control-arm registry (A/B/C/D/G built; E/F/H explicitly deferred, raise NotImplementedError), RecursiveControlArmReportV1 resource accounting (never a parity/winner field), and build_control_arm_table. Extended rng_contract.py with arch_specific: namespaces and RecursiveControlInitializationV1 (fail-closed if common tensors were not actually identically initialized across arms) -- additive to the existing disjoint-namespace contract, not a fork. 19 new tests in tests/test_models/test_recursive_denoiser.py." - }, - { - "version": "v7", - "date": "2026-07-21", - "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v7 revision to embed the new control_arm_table section (SLM-241/RSC-A05) -- content-only refresh of already-committed evidence (verdict/losses/deep-supervision metrics unchanged)." - }, - { - "version": "v6", - "date": "2026-07-21", - "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v6 revision (36dd8d28, code_dirty=false) to embed the new architecture_comparison section/field and the objective-decomposition loss warning -- content-only refresh of already-committed evidence (verdict/losses/deep-supervision metrics unchanged), no further code/gate/harness change." - }, - { - "version": "v6", - "date": "2026-07-21", - "note": "SLM-240 (RSC-A04): retracted recursive_denoiser.py's false 'same parameter count and layer names as DenoiserTower' docstring claim (grepped repo-wide -- only that docstring carried it) and rewrote the module docstring to state precisely what R=1 preserves (public interface / compatible tensor shapes) vs what it never claims (output/parameter equivalence, checkpoint layer-name compatibility for the new z-state keys, any parameter-efficiency claim). Added recursive_zstate_parameter_delta(d_model, max_len) -- the exact formula (z_latent + ctx_proj) reproducing the SLM-138 fixture's 9,248-parameter / +14.23% delta from real constructed towers, never hard-coded -- plus ArchitectureComparisonReportV1/compare_denoiser_architectures (interface_compatible, output_shape_compatible, parameter_count_total/_denoiser, active_parameter_count (grad-based, measured), checkpoint_bytes, common/architecture-specific parameter names+shapes, parameter_count_delta(+formula-match flag), block_evaluations_per_forward (verified against real forward-hook counts), estimated_forward_flops, behaviorally_equivalent_under_declared_degeneracy -- deliberately no single 'parity' field, enforced by __post_init__). Renamed the misnamed test_recursive_steps_one_parity_with_denoiser_tower to test_recursive_r1_preserves_denoiser_interface_and_finite_shapes and added 13 new tests (70 total, all passing) covering the delta formula across (d_model, max_len) pairs, parameter-count independence from recursive_steps, transition-layer name/shape 1:1 mapping, explicit R=1 non-equivalence, checkpoint-migration z-state key coverage, and the comparison report's internal consistency/validation. Wired a real ArchitectureComparisonReportV1 into scripts/run_slm138_recursive_denoiser_fixture.py's report + markdown (interface-compatible/parameter-matched/parameter delta/behavioral parity/claim class fields) and added an objective-decomposition warning (referencing SLM-238's recursive_depth_aux_mode) next to the raw stacked/recursive losses. Annotated docs/design/quality-experiment-matrix.md's V18 section and docs/design/research-lineage.md's SLM-138 row -- SLM-138's wiring-only verdict is unchanged, this is a correction/annotation only. See docs/design/iter-rsc-a04-parity-claim-correction-20260721.{json,md}." - }, - { - "version": "v5", - "date": "2026-07-21", - "note": "no-bump: regenerated iter-rsc-a03-rng-determinism-contract-20260721.{json,md}'s embedded determinism_report from the clean committed revision (4411710e, code_dirty=false throughout) after the v5 harness/doc bump landed -- content-only refresh of already-committed evidence, no further code/gate/harness change." - }, - { - "version": "v5", - "date": "2026-07-21", - "note": "SLM-239 (RSC-A03): added src/slm_training/models/rng_contract.py -- an explicit, disjoint RNG namespace contract (model_initialization/shape_probe_inputs/shape_probe_context/training_corruption/training_batch_order/control_only) plus RngCheckpoint (also covers TwoTowerModel's private self._rng random.Random, a second RNG source _mask_targets reads that a torch-only checkpoint does not capture -- found and fixed during this work). Refactored scripts/run_slm138_recursive_denoiser_fixture.py into the required six-phase execution (construction / deterministic shape probes / deterministic pre-update objective decomposition / one optimizer step / deterministic post-update verification via restored corruption-RNG checkpoint / checkpoint round-trip), added a clean-tree evidence gate (--allow-dirty debug override, non-comparable artifacts never land in docs/design/ without it), and a --mode determinism FixtureDeterminismReportV1 built from real repeated executions + call-order permutations. Added 17 new deterministic regression tests (tests/test_models/test_recursive_denoiser.py, 55 total, all passing) covering the RNG contract, probe-order/extra-probe invariance, restored-checkpoint repeated evaluation (and the torch-only-restore-is-insufficient counter-example), differing-corruption-seed field isolation, fixture exit-state independence from caller entry state, the clean-tree gate, version-stamp match, and checkpoint round-trip digest identity. Measured verdict: bit_exact (real run A/run B/permutation digests match; namespace isolation holds) -- see docs/design/iter-rsc-a03-rng-determinism-contract-20260721.{json,md}." - }, - { - "version": "v4", - "date": "2026-07-21", - "note": "SLM-238 (RSC-A02) follow-up: added test_recursive_objective_contract_v2_validates_sum_identities and a schema-presence assertion in test_generated_decomposition_sums_reproduce_scalar_loss_exactly (41 total). Re-ran scripts/run_rsc_a02_depth_aux_mode_factorial.py --mode fixture --corpus-limit 6 to regenerate the committed docs/design/iter-rsc-a02-depth-aux-mode-factorial-20260721.{json,md} after an earlier --mode plan-only smoke check had accidentally clobbered them with the plan-only skeleton; real fixture-arm numbers now committed and byte-verified against the narrative doc." - }, - { - "version": "v3", - "date": "2026-07-21", - "note": "SLM-238 (RSC-A02): added 14 deterministic regression tests for recursive_depth_aux_mode/recursive_depth_aux_weight and the new objective-decomposition telemetry (tests/test_models/test_recursive_denoiser.py, 40 total, all passing). Added scripts/run_rsc_a02_depth_aux_mode_factorial.py -- a bounded 5-arm (A off / B intermediate_only+uniform / C all_depths+uniform / D intermediate_only+lambda=0.3 / E all_depths+lambda=0.3) deterministic-fixture + bounded real-corpus-smoke comparison; calibration/semantics only, no quality or promotion claim." - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "SLM-237 (RSC-A01): added 17 deterministic regression tests for the corrected weighted deep-supervision objective and the new validate_recursive_depth_supervision fail-closed validator (tests/test_models/test_recursive_denoiser.py). Fixed scripts/run_slm138_recursive_denoiser_fixture.py's own latent instance of failure mode #6 -- it applied recursive_depth_supervision_weights=(0.5, 1.0) to the 'stacked' arm too, which the old code silently ignored and the new fail-closed validator correctly rejects; now only the 'shared_recursive' arm sets the weights." - }, - { - "version": "v2", - "date": "2026-07-21", - "note": "SLM-211: add tie_output_embedding control to SharedRecursiveDenoiserTower" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" - }, - { - "version": "v1", - "date": "2026-07-20", - "note": "initial registration; SLM-138 shared recursive denoiser tower wiring/fixture harness" - } - ] - }, - "model.twotower": { - "version": "v234", - "kind": "harness", - "paths": [ - "src/slm_training/models/twotower.py", - "src/slm_training/models/semantic_connector.py", - "src/slm_training/models/dynamic_pointer_scorer.py", - "src/slm_training/models/choice_tokenizer.py", - "src/slm_training/models/tokenizer.py", - "src/slm_training/models/grammar.py", - "src/slm_training/models/onnx_inference.py", - "src/slm_training/models/checkpoint_migrate.py", - "src/slm_training/models/template_fill.py", - "src/slm_training/dsl/grammar/fastpath/compiler_draft.py", - "src/slm_training/dsl/grammar/fastpath/token_map.py", - "src/slm_training/data/quality.py", - "src/slm_training/data/contract.py", - "src/slm_training/models/adapters/", - "src/slm_training/harnesses/model_build/config.py", - "src/slm_training/harnesses/model_build/factory.py", - "scripts/train_model.py", - "src/slm_training/models/twotower_numeric_gates.py", - "tests/test_models/test_twotower_numeric_gates.py", - "docs/design/iter-slm242-twotower-numeric-gates-20260721.json", - "docs/design/iter-slm242-twotower-numeric-gates-20260721.md" - ], - "history": [ - { - "version": "v234", - "date": "2026-07-24", - "note": "no-bump: add tree-edit checkpoint format migration without changing TwoTower behavior" - }, - { - "version": "v234", - "date": "2026-07-24", - "note": "Persist exact parameter count and serialized FP32 weight-byte estimate in TwoTower checkpoint metadata" - }, - { - "version": "v233", - "date": "2026-07-24", - "note": "expose existing decode-time model-build levers through the train CLI so periodic evaluation receives the configured values" - }, - { - "version": "v232", - "date": "2026-07-24", - "note": "require budgeted pack-owned completion domains for compiler tree, LTR, and one-hole constrained decoding" - }, - { - "version": "v231", - "date": "2026-07-23", - "note": "persist and validate SLM-243 recurrence update, empty-F, and norm modes while migrating old checkpoints to historical defaults" - }, - { - "version": "v231", - "date": "2026-07-23", - "note": "serve compiler-tree slot contracts through runtime feature tables so prompt-derived OpenUI template markers stay the model output contract" - }, - { - "version": "v230", - "date": "2026-07-23", - "note": "Constrain configured optimizers to the supported AdamW and scratch-only Muon hybrid choices" - }, - { - "version": "v229", - "date": "2026-07-23", - "note": "judge symbolic Harness identity rows from their parsed payload instead of the retired English input delimiter" - }, - { - "version": "v228", - "date": "2026-07-23", - "note": "Expose explicit staged capability, prior-certificate, and distillation preflight inputs through model-build configuration and train CLI" - }, - { - "version": "v227", - "date": "2026-07-23", - "note": "Plumb immutable campaign manifest, result, store, and artifact roots through explicit model-promotion configuration" - }, - { - "version": "v226", - "date": "2026-07-23", - "note": "SLM-279: require new configs with non-empty recursive-depth weights to name intermediate_only, all_depths, or legacy_all_depths explicitly; keep empty configs primary-only and migrate pre-field checkpoint/config loads to legacy_all_depths, including tokenizer and stacked-to-recursive migration paths" - }, - { - "version": "v225", - "date": "2026-07-23", - "note": "no-bump: extract the existing normalized recursive-depth loss arithmetic into a pure reducer without changing objective semantics or telemetry" - }, - { - "version": "v225", - "date": "2026-07-22", - "note": "Constrain primitive array items and separators from their generated schema type while preserving component-array and positional-call boundaries" - }, - { - "version": "v224", - "date": "2026-07-22", - "note": "Keep lexer-native numeric literals inside a schema-checked byte automaton and share one token-to-source conversion across compiler and persistent grammar state" - }, - { - "version": "v223", - "date": "2026-07-22", - "note": "Derive component-array boundaries, nonempty typed children, available-content closure, typed binder compatibility, and acyclic references from schema so constrained decode cannot enter impossible continuations" - }, - { - "version": "v222", - "date": "2026-07-22", - "note": "Derive certified fallback markers from the request contract and use a marker-free fallback when no contract exists" - }, - { - "version": "v221", - "date": "2026-07-22", - "note": "Centralize tokenizer sidecar dispatch and preserve lexer-native tokenizer identity through checkpoint migration" - }, - { - "version": "v220", - "date": "2026-07-22", - "note": "Make declared lexer template markers single-use under slot-contract constrained decode" - }, - { - "version": "v219", - "date": "2026-07-22", - "note": "Bind only direct semantic-role obligations globally so nested repeated-owner markers cannot displace standalone siblings" - }, - { - "version": "v218", - "date": "2026-07-22", - "note": "no-bump: add evaluator-only record offset metadata to ModelBuildConfig without changing model construction or decoding" - }, - { - "version": "v218", - "date": "2026-07-22", - "note": "Share marker namespace ownership between semantic obligations and repeated decode so exhausted structural owners add standalone siblings" - }, - { - "version": "v217", - "date": "2026-07-22", - "note": "Scope repeated-instance ownership to structurally open lexer component calls so parent sibling arrays remain open" - }, - { - "version": "v216", - "date": "2026-07-22", - "note": "Apply repeated-instance marker ownership during restricted and compiler-tree path ranking" - }, - { - "version": "v215", - "date": "2026-07-22", - "note": "Use the shared semantic-plan margin to keep template-marker namespaces owned by their repeated lexer component instance" - }, - { - "version": "v214", - "date": "2026-07-22", - "note": "Carry the semantic-plan margin across lexer array delimiters while sibling component requirements remain" - }, - { - "version": "v213", - "date": "2026-07-22", - "note": "Exclude role metadata from component cardinality and preserve repeated lexer families as siblings unless explicit outer-group topology permits nesting" - }, - { - "version": "v212", - "date": "2026-07-22", - "note": "Treat planned container families as satisfying joint semantic roles reachable through their schema descendants" - }, - { - "version": "v211", - "date": "2026-07-22", - "note": "Apply semantic-plan family scores to direct lexer component candidates inside compiler containers" - }, - { - "version": "v210", - "date": "2026-07-22", - "note": "no-bump: keep compiler test fixtures valid under the stricter central slot-inventory companion contract; model behavior is unchanged" - }, - { - "version": "v210", - "date": "2026-07-22", - "note": "Continue lexer compiler calls into an optional schema property when it owns a still-missing bound semantic role" - }, - { - "version": "v209", - "date": "2026-07-22", - "note": "Apply the existing required-slot margin through one shared compiler restricted/tree adapter" - }, - { - "version": "v208", - "date": "2026-07-22", - "note": "Normalize real lexer component paths as bound families for semantic-plan scoring inside an existing container" - }, - { - "version": "v207", - "date": "2026-07-22", - "note": "Floor an immediate lexer compiler container close after every declared template symbol is consumed" - }, - { - "version": "v206", - "date": "2026-07-22", - "note": "no-bump: consume the canonical slot-contract lever set in the retained runtime guard without changing decode behavior" - }, - { - "version": "v206", - "date": "2026-07-22", - "note": "Treat lexer compiler component paths as bound component candidates in the shared slot-family scorer" - }, - { - "version": "v205", - "date": "2026-07-22", - "note": "Pass declared semantic-role candidates through the shared slot-family scorer on lexer compiler paths" - }, - { - "version": "v204", - "date": "2026-07-22", - "note": "Rank lexer template symbols by the active grammar call and public schema property at every compiler trie edge" - }, - { - "version": "v203", - "date": "2026-07-22", - "note": "Apply semantic-plan margin to keep planned lexer typed-array subtrees nonempty" - }, - { - "version": "v202", - "date": "2026-07-22", - "note": "Count fulfilled semantic-plan families by tokenizer component IDs so bare lexer symbols cannot be forced repeatedly" - }, - { - "version": "v201", - "date": "2026-07-22", - "note": "Keep topology-free lexer semantic-plan obligations on bound families so they cannot override the trained root-role decision" - }, - { - "version": "v200", - "date": "2026-07-22", - "note": "Apply prompt-semantic component obligations during lexer compiler path selection instead of silently ignoring the configured plan" - }, - { - "version": "v199", - "date": "2026-07-22", - "note": "Build prompt-semantic component obligations for the lexer compiler instead of rejecting its supported scoring path" - }, - { - "version": "v198", - "date": "2026-07-22", - "note": "no-bump: record the matched E735 training and smoke evidence for the full-head root-arity objective" - }, - { - "version": "v198", - "date": "2026-07-22", - "note": "Train root-reference arity against the full inference head so high-arity tail logits receive negative supervision" - }, - { - "version": "v197", - "date": "2026-07-22", - "note": "Reject rank-calibrated lexer root arity after it caused three of three bounded smoke decodes to time out" - }, - { - "version": "v196", - "date": "2026-07-22", - "note": "Calibrate lexer root-reference arity weight one as interpolation to the trained continue-or-stop class" - }, - { - "version": "v195", - "date": "2026-07-22", - "note": "Remove unreachable lexer root-reference identity ranking while retaining strict serving checkpoint validation for enabled root heads" - }, - { - "version": "v194", - "date": "2026-07-22", - "note": "Add lexer-native root-reference identity supervision and compiler-path ranking; reject serving checkpoints missing enabled root heads" - }, - { - "version": "v193", - "date": "2026-07-22", - "note": "Fail closed when a learned decode head is enabled without a nonzero owning checkpoint training objective" - }, - { - "version": "v192", - "date": "2026-07-22", - "note": "Add lexer-native root-reference arity supervision and root-list continue-stop scoring" - }, - { - "version": "v191", - "date": "2026-07-21", - "note": "no-bump: adjust ModelBuildConfig policy regression expectations; TwoTower model behavior is unchanged" - }, - { - "version": "v191", - "date": "2026-07-21", - "note": "Apply atomic evaluation policy presets during ModelBuildConfig construction" - }, - { - "version": "v190", - "date": "2026-07-22", - "note": "Fail closed when enabled decode levers cannot execute for the selected output tokenizer and compiler mode" - }, - { - "version": "v189", - "date": "2026-07-22", - "note": "E726: fail closed when choice-only root-reference levers are requested with lexer or compositional output" - }, - { - "version": "v188", - "date": "2026-07-21", - "note": "E716: keep non-string literal markers out of schema-typed STRING compiler candidates" - }, - { - "version": "v187", - "date": "2026-07-21", - "note": "centralize model and training levers while reducing the canonical wall-budget default and maximum to two minutes" - }, - { - "version": "v186", - "date": "2026-07-21", - "note": "Enforce symbol-only output tokenization, remove the free-form LIT_STR channel, and invalidate pre-v2 output-contract checkpoints" - }, - { - "version": "v185", - "date": "2026-07-21", - "note": "E713: compile explicit group-inside-outer-component prompt topology into a verified nested root plan" - }, - { - "version": "v184", - "date": "2026-07-21", - "note": "E712: preserve explicit component counts across arbitrary descriptive modifiers without borrowing counts across phrase boundaries" - }, - { - "version": "v183", - "date": "2026-07-21", - "note": "E711: avoid unprompted joint role carriers when the prompt-planned families can cover the full role group within schema capacity" - }, - { - "version": "v182", - "date": "2026-07-21", - "note": "E709: finalize enum-slot protection after semantic biases and close fully covered terminal Stack lists by margin" - }, - { - "version": "v181", - "date": "2026-07-21", - "note": "add opaque request-local identity to opt-in bound/choice generation while preserving historical name-aware checkpoint semantics and retained v180 carrier behavior; exclude projected marker names from scoring while allowing declared typed authority; add verified exact-choice results, persistent typed scope primitives, and narrow one-hole deterministic MaskGIT completion" - }, - { - "version": "v180", - "date": "2026-07-21", - "note": "E708: retain only a newly created missing-slot carrier in terminal Stack aggregation" - }, - { - "version": "v179", - "date": "2026-07-21", - "note": "E707: restore v177 behavior after terminal-root completeness improved Rico coverage but regressed adversarial fidelity" - }, - { - "version": "v178", - "date": "2026-07-21", - "note": "E707: pair one bounded missing-slot carrier with complete terminal-root reference aggregation" - }, - { - "version": "v177", - "date": "2026-07-21", - "note": "E706: restore v175 behavior after bounded missing-slot carrier routing was neutral across the five-suite replay" - }, - { - "version": "v176", - "date": "2026-07-21", - "note": "E706: bind one still-missing visible slot to a legal direct carrier before verified semantic-plan root closure" - }, - { - "version": "v175", - "date": "2026-07-21", - "note": "E703 rejection: restore v173 repeated-slot scoring after enum-property abstention left all 19 matched predictions unchanged" - }, - { - "version": "v174", - "date": "2026-07-21", - "note": "E703: prevent repeated-instance visible-slot flooring from overriding enum-valued component properties" - }, - { - "version": "v173", - "date": "2026-07-21", - "note": "E702 rejection: restore v171 joint-role planning after counted Callout siblings displaced planned Cards and regressed Held-out and Rico structure" - }, - { - "version": "v172", - "date": "2026-07-21", - "note": "E702: count another preferred joint-role carrier when existing instances cannot cover the accumulated namespace roles" - }, - { - "version": "v171", - "date": "2026-07-21", - "note": "E701 r6: abstain from capacity when roles lack direct property matches and preserve carriers under explicitly typed container descendants" - }, - { - "version": "v170", - "date": "2026-07-21", - "note": "E701 r5: materialize a replacement role carrier only when a directly compatible planned carrier is capacity-exhausted, preserving ordinary nested reachability" - }, - { - "version": "v169", - "date": "2026-07-21", - "note": "E701 r2: materialize enough counted instances for reachable role carriers instead of leaving their bindings advisory" - }, - { - "version": "v168", - "date": "2026-07-21", - "note": "E701: enforce distinct compatible string-property capacity for jointly introduced role carriers and cover the actual broad Form candidate set" - }, - { - "version": "v167", - "date": "2026-07-21", - "note": "E700 rejection: restore v165 role planning after property-capacity matching changed the plan but left all emitted predictions and aggregate quality unchanged" - }, - { - "version": "v166", - "date": "2026-07-21", - "note": "E700: require existing planned component instances to have distinct compatible public string properties before binding another visible role" - }, - { - "version": "v165", - "date": "2026-07-21", - "note": "Merge deterministic active-runtime completion from v94 with the retained E698 likeness abstention and role-planning behavior" - }, - { - "version": "v164", - "date": "2026-07-21", - "note": "Merge the non-finite bound-role mask guard with the retained E698 likeness abstention and enum-role disambiguation behavior" - }, - { - "version": "v163", - "date": "2026-07-21", - "note": "E699 rejection: restore v159 role candidates after form aliases destabilized joint role planning" - }, - { - "version": "v162", - "date": "2026-07-21", - "note": "E699 rejection: restore pre-capacity role planning after the revisit destabilized Form role ownership; retain only explicit form-role alias alignment" - }, - { - "version": "v161", - "date": "2026-07-21", - "note": "E699 correction: align form-role aliases so capacity planning recognizes placeholder/value string properties on public input controls" - }, - { - "version": "v160", - "date": "2026-07-21", - "note": "E699: bound planned-family role reuse by distinct public string-property capacity after likeness abstention removes the false Form owner" - }, - { - "version": "v159", - "date": "2026-07-21", - "note": "E698 r2 correction: disambiguate a visible semantic role when exactly one compatible family names it in a public enum" - }, - { - "version": "v158", - "date": "2026-07-21", - "note": "E698: exclude adjectival component-like mentions from the shared prompt semantic plan" - }, - { - "version": "v157", - "date": "2026-07-21", - "note": "E697 r2 correction: defer ambiguous aggregate families until concrete remaining planned families are emitted" - }, - { - "version": "v156", - "date": "2026-07-21", - "note": "E697: order a planned parent before planned families reachable through its required non-alternative schema paths" - }, - { - "version": "v155", - "date": "2026-07-21", - "note": "E696 rejection: restore v152 behavior after scalar literal fallback left Form placeholder spam unchanged and generated arbitrary operational names" - }, - { - "version": "v154", - "date": "2026-07-21", - "note": "E696 r2 correction: prefer the live grammar's framed string literal before its unavailable fixed empty spelling" - }, - { - "version": "v153", - "date": "2026-07-21", - "note": "E696: route a repeated or incompatible visible slot to the legal empty literal when no unused role-compatible scalar string slot remains" - }, - { - "version": "v152", - "date": "2026-07-21", - "note": "E695 rejection: restore v150 behavior after property-capacity planning only swapped title roles and left placeholder spam, strict meaning, and aggregate quality unchanged" - }, - { - "version": "v151", - "date": "2026-07-21", - "note": "E695: reuse a planned family for another visible role only when distinct public string properties can still bind every assigned role" - }, - { - "version": "v150", - "date": "2026-07-21", - "note": "E694: finalize invalid enum literals with a framed open-vocabulary spelling when the generated row has capacity" - }, - { - "version": "v149", - "date": "2026-07-21", - "note": "E693: plan a public Group component from its unique base noun when no standalone base component exists" - }, - { - "version": "v148", - "date": "2026-07-21", - "note": "E692 r2 correction: treat compact direction tokens as semantically valid enum spellings while repairing invalid fixed literals" - }, - { - "version": "v147", - "date": "2026-07-21", - "note": "E692: normalize invalid fixed enum literals after decode while preserving already-valid enum choices and later decisions" - }, - { - "version": "v146", - "date": "2026-07-21", - "note": "E691: keep transitively slot-bearing array content schema-constrained instead of admitting a raw placeholder" - }, - { - "version": "v145", - "date": "2026-07-21", - "note": "E690: stop schema-reachable wrapper continuation after all of that component's explicit planned role bindings are covered" - }, - { - "version": "v144", - "date": "2026-07-21", - "note": "E689 r2 correction: preserve established schema-role binding at positional slots while testing planned parent descendant disambiguation" - }, - { - "version": "v143", - "date": "2026-07-21", - "note": "E689: disambiguate visible role carriers through cycle-safe schema descendants of already planned parent families; restore E688 wrapper behavior" - }, - { - "version": "v142", - "date": "2026-07-21", - "note": "E688: prevent schema-reachable wrapper continuation from stealing missing roles outside that component's explicit semantic-plan bindings" - }, - { - "version": "v141", - "date": "2026-07-21", - "note": "E686 rejection: restore the original bounded per-call trace budget after model-local rows could not identify independently generated eval records" - }, - { - "version": "v140", - "date": "2026-07-21", - "note": "E686: bound constrained-selection telemetry per batch row so early examples cannot starve later failure traces" - }, - { - "version": "v139", - "date": "2026-07-21", - "note": "E685: plan a non-house-style carrier when it is the sole schema-compatible family for a visible role" - }, - { - "version": "v138", - "date": "2026-07-21", - "note": "E684: derive singular prompt matches for plural schema component families without multiplying singular compound containers" - }, - { - "version": "v137", - "date": "2026-07-21", - "note": "E682: constrain explicitly planned positional role slots by their active public-schema property name" - }, - { - "version": "v136", - "date": "2026-07-21", - "note": "E681: partition larger visible-role namespaces into maximal disjoint groups covered by specific direct-schema carriers" - }, - { - "version": "v135", - "date": "2026-07-21", - "note": "E680: map visible kicker and heading display roles to schema-valid TextContent carriers" - }, - { - "version": "v134", - "date": "2026-07-21", - "note": "E678 provenance correction: localize E676 joint-role planning inside TwoTower so model-only helpers do not perturb the binding-aware evaluator implementation hash; decode behavior is unchanged" - }, - { - "version": "v133", - "date": "2026-07-21", - "note": "E676: plan one schema component for same-namespace roles only when distinct direct string properties jointly cover them" - }, - { - "version": "v132", - "date": "2026-07-21", - "note": "E675: prefer true at a boolean open property only for the actively authored component family" - }, - { - "version": "v131", - "date": "2026-07-21", - "note": "E674: align visible semantic-role candidates with canonical action and dashboard display aliases for Button.label, Callout.description, and TextContent.text" - }, - { - "version": "v130", - "date": "2026-07-21", - "note": "E673: resolve cycle-safe public schema references when testing whether typed-array items can reach visible slots" - }, - { - "version": "v129", - "date": "2026-07-21", - "note": "E672 rejection: restore retained E666 behavior after public-reference item schemas kept slot reachability inactive" - }, - { - "version": "v128", - "date": "2026-07-21", - "note": "E672: let active typed-array schemas start a minimal slot-bearing item even when the enclosing component was model-introduced" - }, - { - "version": "v127", - "date": "2026-07-21", - "note": "E671 rejection: restore retained E666 behavior after nearest nested ownership remained neutral because the real Carousel was not authored in the semantic plan" - }, - { - "version": "v126", - "date": "2026-07-21", - "note": "E671: combine nested item schemas and role-aware targets with nearest-component ownership through nested arrays" - }, - { - "version": "v125", - "date": "2026-07-21", - "note": "E670 rejection: restore retained E666 behavior after the combined nested-schema and role-aware wrapper policy reproduced E669's wrong Form wrapper" - }, - { - "version": "v124", - "date": "2026-07-21", - "note": "E670: combine nested array schema propagation with schema-allowed semantic-role wrapper selection" - }, - { - "version": "v123", - "date": "2026-07-21", - "note": "E669 rejection: restore retained E666 behavior after nested schema propagation chose a semantically wrong Form wrapper and regressed structure, AST F1, and latency" - }, - { - "version": "v122", - "date": "2026-07-21", - "note": "E669: propagate active array item schemas through nested list frames so inner values remain schema-constrained" - }, - { - "version": "v121", - "date": "2026-07-21", - "note": "E668 rejection: restore retained E666 behavior after schema-role typed-array wrappers produced identical predictions and quality metrics" - }, - { - "version": "v120", - "date": "2026-07-21", - "note": "E668: prefer schema-allowed semantic-role component wrappers for missing visible slots at empty typed-array items" - }, - { - "version": "v119", - "date": "2026-07-21", - "note": "E667 rejection: restore retained E666 behavior after nested typed-array ownership produced identical predictions and quality metrics" - }, - { - "version": "v118", - "date": "2026-07-21", - "note": "E667: apply schema-derived typed-array item margins through nested arrays owned by an authored plan component" - }, - { - "version": "v117", - "date": "2026-07-21", - "note": "E666: normalize completed dynamic literals at schema-enum properties after choice decoding so later model decisions remain unchanged" - }, - { - "version": "v116", - "date": "2026-07-21", - "note": "E659 rejection: restore retained E653 behavior after hard property-role masking collapsed meaningfulness, fidelity, structure, reward, and latency" - }, - { - "version": "v115", - "date": "2026-07-21", - "note": "E659: final-mask visible slots that violate the active positional property's schema role" - }, - { - "version": "v114", - "date": "2026-07-21", - "note": "E658 rejection: restore retained E653 behavior after property-aware margins dropped a required metric and regressed fidelity, structure, and reward" - }, - { - "version": "v113", - "date": "2026-07-21", - "note": "E658: gate raw visible-slot margins by the active positional property's schema role" - }, - { - "version": "v112", - "date": "2026-07-21", - "note": "E657 rejection: restore retained E653 behavior after combined ownership constraints regressed structure, AST F1, and latency without improving strict meaning" - }, - { - "version": "v111", - "date": "2026-07-21", - "note": "E657: combine direct-owner raw-slot constraints across coverage and repeated-plan margins" - }, - { - "version": "v110", - "date": "2026-07-21", - "note": "E656 rejection: restore retained E653 behavior after repeated-slot role ownership alone left predictions and quality unchanged" - }, - { - "version": "v109", - "date": "2026-07-21", - "note": "E656: restrict repeated-plan raw-slot margins to roles directly owned by the active component" - }, - { - "version": "v108", - "date": "2026-07-21", - "note": "E655 rejection: restore retained E653 behavior after direct-owner slot forcing left the matched predictions and quality metrics unchanged" - }, - { - "version": "v107", - "date": "2026-07-21", - "note": "E655: require direct semantic-role ownership before forcing a raw visible slot; transitive owners descend through compatible children" - }, - { - "version": "v106", - "date": "2026-07-21", - "note": "E654 rejection: restore retained E653 v104 after enum flooring reduced fidelity, validity, structure, and reward" - }, - { - "version": "v105", - "date": "2026-07-21", - "note": "E654: retry schema-valid enum flooring after E653 stabilizes nested visible-role ownership" - }, - { - "version": "v104", - "date": "2026-07-21", - "note": "E653: keep visible-role leaf obligations nested when an already-planned component can reach the role through its public schema" - }, - { - "version": "v103", - "date": "2026-07-21", - "note": "Restore v98 behavior after E652 detached metric leaves from their planned Cards and reduced meaningful output." - }, - { - "version": "v102", - "date": "2026-07-21", - "note": "Recognize display value roles as text-compatible so planned containers receive simple visible leaf owners." - }, - { - "version": "v101", - "date": "2026-07-21", - "note": "Restore v96 behavior after E651 enum flooring reduced meaningful output and produced an empty planned Card." - }, - { - "version": "v100", - "date": "2026-07-21", - "note": "Floor schema-valid enum literals or optional closure above invalid dynamic values during constrained decode." - }, - { - "version": "v99", - "date": "2026-07-21", - "note": "Bind compatible visible roles to already-planned families before inferring any additional family instances." - }, - { - "version": "v98", - "date": "2026-07-21", - "note": "Restore v93 behavior after E649 refresh-role binding reduced meaningful output quality despite complete placeholder coverage." - }, - { - "version": "v97", - "date": "2026-07-21", - "note": "Recognize refresh as an action-label semantic role so prompt-planned controls can bind their visible slot." - }, - { - "version": "v96", - "date": "2026-07-20", - "note": "E648: verify semantic-root closures through the dynamic-literal-aware choice tokenizer before official parser validation" - }, - { - "version": "v95", - "date": "2026-07-20", - "note": "E647 r2: deduplicate semantic-root abstention traces by stable error signature rather than changing planned-section counts" - }, - { - "version": "v94", - "date": "2026-07-20", - "note": "E647: emit bounded behavior-neutral semantic-root verifier abstention evidence with error type, message, and planned closure counts" - }, - { - "version": "v93", - "date": "2026-07-20", - "note": "E646 rejection: restore v88 plan-counted root references after all-section reachability closure was prediction- and quality-neutral" - }, - { - "version": "v92", - "date": "2026-07-20", - "note": "E646: close verified semantic roots over every completed top-level element after required plan coverage so unplanned valid sections cannot orphan the root" - }, - { - "version": "v91", - "date": "2026-07-20", - "note": "E644: pair visible-role-inferred families with concrete unused slots and margin-floor only the assigned role at compatible schema positions" - }, - { - "version": "v90", - "date": "2026-07-20", - "note": "E643 rejection: restore v85 prompt-only semantic plans after direct required-string role binding reproduced E641 strict, structure, and latency regressions" - }, - { - "version": "v89", - "date": "2026-07-20", - "note": "E643: bind visible-role-inferred families to compatible required plain-string content arguments while preserving pre-content operational literals" - }, - { - "version": "v88", - "date": "2026-07-20", - "note": "E642 rejection: restore v83 prompt-only semantic plans after root-only inferred families regressed meaning, structure, reward, AST F1, and tail latency" - }, - { - "version": "v87", - "date": "2026-07-20", - "note": "E642: constrain visible-role-inferred semantic-plan families to completed root-section boundaries to prevent nested family spam" - }, - { - "version": "v86", - "date": "2026-07-20", - "note": "E641 rejection: restore v84 prompt-only semantic plans after visible-role completion cut strict v2, structure, and latency despite fidelity and reward gains" - }, - { - "version": "v85", - "date": "2026-07-20", - "note": "E641: complete prompt-planned families with house-style-preferred schema-compatible leaves for visible roles no planned family can own" - }, - { - "version": "v84", - "date": "2026-07-20", - "note": "E640 rejection: restore v82 state and root references after supplemental slot-bearing reference tracking left every policy-matched output and metric unchanged" - }, - { - "version": "v83", - "date": "2026-07-20", - "note": "E640: track visible slots per completed root section and include supplemental slot-bearing sections in verifier-checked semantic root references" - }, - { - "version": "v82", - "date": "2026-07-20", - "note": "E639 rejection: restore v80 root-boundary behavior after sibling coverage sampling changed traces but the verified root omitted those unplanned sections and every policy-matched metric stayed flat" - }, - { - "version": "v81", - "date": "2026-07-20", - "note": "E639: continue from completed root sections with the best legal role-compatible component while visible slots remain" - }, - { - "version": "v94", - "date": "2026-07-21", - "note": "make exact LTR, repair, choice, and complete-compiler singleton decisions bypass neural rows; add honest verified template-plus-bindings generation boundary" - }, - { - "version": "v93", - "date": "2026-07-21", - "note": "Skip role-obligation margin flooring for candidates already blocked to a non-finite score by semantic candidate masks" - }, - { - "version": "v92", - "date": "2026-07-21", - "note": "no-bump: renumber E644 role-obligation evidence to E650 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" - }, - { - "version": "v92", - "date": "2026-07-20", - "note": "E650: pair visible-role-inferred families with concrete unused slots and margin-floor only the assigned role at compatible schema positions; renumbered from the originally launched E644 arm after rebase" - }, - { - "version": "v91", - "date": "2026-07-21", - "note": "no-bump: renumber E643 bound-role-plan evidence to E649 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" - }, - { - "version": "v91", - "date": "2026-07-20", - "note": "E649 rejection: restore v89 prompt-only semantic plans after direct required-string role binding reproduced E647 strict, structure, and latency regressions; renumbered from the originally launched E643 arm after rebase" - }, - { - "version": "v90", - "date": "2026-07-20", - "note": "E649: bind visible-role-inferred families to compatible required plain-string content arguments while preserving pre-content operational literals; renumbered from the originally launched E643 arm after rebase" - }, - { - "version": "v89", - "date": "2026-07-21", - "note": "no-bump: renumber E642 root-only-role-plan evidence to E648 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" - }, - { - "version": "v89", - "date": "2026-07-20", - "note": "E648 rejection: restore v87 prompt-only semantic plans after root-only inferred families regressed meaning, structure, reward, AST F1, and tail latency; renumbered from the originally launched E642 arm after rebase" - }, - { - "version": "v88", - "date": "2026-07-20", - "note": "E648: constrain visible-role-inferred semantic-plan families to completed root-section boundaries to prevent nested family spam; renumbered from the originally launched E642 arm after rebase" - }, - { - "version": "v87", - "date": "2026-07-21", - "note": "no-bump: renumber E641 role-plan-completion evidence to E647 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" - }, - { - "version": "v87", - "date": "2026-07-20", - "note": "E647 rejection: restore v85 prompt-only semantic plans after visible-role completion cut strict v2, structure, and latency despite fidelity and reward gains; renumbered from the originally launched E641 arm after rebase" - }, - { - "version": "v86", - "date": "2026-07-20", - "note": "E647: complete prompt-planned families with house-style-preferred schema-compatible leaves for visible roles no planned family can own; renumbered from the originally launched E641 arm after rebase" - }, - { - "version": "v85", - "date": "2026-07-21", - "note": "no-bump: reconcile E646 evidence to treatment v84 and restoration v85 after integrating SLM-242 and E645 through v83; no model behavior change" - }, - { - "version": "v85", - "date": "2026-07-20", - "note": "E646 rejection: restore v83 state and root references after supplemental slot-bearing reference tracking left every policy-matched output and metric unchanged; renumbered from the originally launched E640 arm after rebase" - }, - { - "version": "v84", - "date": "2026-07-21", - "note": "no-bump: renumber E640 root-slot-reference evidence to E646 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" - }, - { - "version": "v84", - "date": "2026-07-20", - "note": "E646: track visible slots per completed root section and include supplemental slot-bearing sections in verifier-checked semantic root references; renumbered from the originally launched E640 arm after rebase" - }, - { - "version": "v83", - "date": "2026-07-21", - "note": "no-bump: renumber E639 sibling-coverage evidence to E645 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" - }, - { - "version": "v83", - "date": "2026-07-20", - "note": "E645 rejection: restore v81 numeric-gate and root-boundary behavior after sibling coverage sampling changed traces but the verified root omitted those unplanned sections and every policy-matched metric stayed flat" - }, - { - "version": "v82", - "date": "2026-07-20", - "note": "E645: continue from completed root sections with the best legal role-compatible component while visible slots remain; renumbered from the originally launched E639 arm after rebase" - }, - { - "version": "v81", - "date": "2026-07-21", - "note": "no-bump: validator refinements after broader test run -- allow steps=0, decode_min_content=-1, finite (not non-negative) slot_component_lexeme/span priors, and a human-readable max_wall_minutes message; no change to the fail-closed intent or to successfully validated configs" - }, - { - "version": "v81", - "date": "2026-07-21", - "note": "SLM-242 (RSC-A06): fail-closed numeric/schedule validation gate for every TwoTower weight and schedule vector; adds src/slm_training/models/twotower_numeric_gates.py, ModelBuildConfig.__post_init__ numeric gate, TwoTowerConfig.__post_init__ numeric gate, CLI --recursive-depth-aux-mode/--recursive-depth-aux-weight in scripts/train_model.py" - }, - { - "version": "v80", - "date": "2026-07-21", - "note": "no-bump: integrate PR #625 version history for model.twotower; prior PR entries below record the behavior changes" - }, - { - "version": "v69", - "date": "2026-07-21", - "note": "SLM-241 (RSC-A05) second follow-up: added recursive_detach_between_steps: bool = False to TwoTowerConfig (and ModelBuildConfig / _twotower_config_from_build / scripts/train_model.py --recursive-detach-between-steps) -- an orthogonal flag to denoiser_arch/z_state_mode, threaded into SharedRecursiveDenoiserTower(detach_between_steps=...) in the shared_recursive construction branch. Reuses denoiser_arch='shared_recursive' (arm B's/G's arch string; no new value); when True, this is control arm H (stop-gradient recurrence). No change to any other denoiser_arch's behavior; existing shared_recursive checkpoint z_latent/ctx_proj warm-start allowance already covers H (same parameter names as B)." - }, - { - "version": "v68", - "date": "2026-07-21", - "note": "SLM-241 (RSC-A05) follow-up: added stacked_matched_state as a canonical denoiser_arch value (control arm E, stacked + matched state capacity) -- routes to the new StackedMatchedStateDenoiserTower (src/slm_training/models/recursive_denoiser.py): an unshared, non-recursive DenoiserTower variant (denoiser_layers blocks, each called once -- same block-evaluation count as arm A) plus a learned state/state_ctx_proj pair shape-matched to arm B's z_latent/ctx_proj, injected once before the transition blocks run (never recurrently re-applied). Checkpoint loader gains a matching warm-start allowance for denoiser.state/denoiser.state_ctx_proj.* missing keys, mirroring the existing shared_recursive allowance. No change to existing 'stacked'/'stacked_depth_matched'/'shared_recursive'/'shared_recursive_y_only'/'shared_recursive_no_extra_capacity' behavior. CLI --denoiser-arch (scripts/train_model.py) gains the new choice." - }, - { - "version": "v67", - "date": "2026-07-21", - "note": "SLM-241 (RSC-A05) follow-up: added stacked_depth_matched as a canonical denoiser_arch value (control arm F, unshared depth-matched tower) -- routes to a plain DenoiserTower built with recursive_steps * recursive_transition_layers independent transition blocks instead of denoiser_layers, so its block-evaluation count matches the shared-recursive arm's (arm B) exactly. No new tower class; no z-state; no change to existing 'stacked'/'shared_recursive'/'shared_recursive_y_only'/'shared_recursive_no_extra_capacity' behavior. CLI --denoiser-arch (scripts/train_model.py) gains the new choice." - }, - { - "version": "v66", - "date": "2026-07-21", - "note": "SLM-241 (RSC-A05): denoiser construction now fails closed on an unrecognized denoiser_arch (KNOWN_DENOISER_ARCHES) instead of silently falling back to 'stacked' -- added shared_recursive_y_only and shared_recursive_no_extra_capacity as canonical denoiser_arch values (control arms C/D), routed through SharedRecursiveDenoiserTower's new z_state_mode via SHARED_RECURSIVE_ARCH_Z_STATE_MODES. CLI --denoiser-arch (scripts/train_model.py) gains the two new choices. No change to existing 'stacked'/'shared_recursive' behavior." - }, - { - "version": "v65", - "date": "2026-07-21", - "note": "SLM-238 (RSC-A02) follow-up: added the required RecursiveObjectiveContractV2 schema (src/slm_training/models/twotower.py) -- a typed, validated view over the objective-decomposition telemetry (mode, aux_weight, primary/intermediate/final/combined loss terms) whose __post_init__ re-checks both required sum identities at construction time. training_loss now also populates last_training_metrics['recursive_objective_contract'] (a dict) alongside the existing flat scalar fields, which remain the single source of truth. Additive only -- no existing metric value, gate, or default changed." - }, - { - "version": "v64", - "date": "2026-07-21", - "note": "SLM-238 (RSC-A02): added recursive_depth_aux_mode (off | intermediate_only | all_depths | legacy_all_depths) and recursive_depth_aux_weight to TwoTowerConfig, extending validate_recursive_depth_supervision/ValidatedDepthSupervision (not forking it) to accept mode/expected-depth-range. Makes the final recursion depth's double-counting (rec_out['logits'] == rec_out['depth_logits'][-1], so the primary term already reflects it once) an explicit, named choice: 'intermediate_only' structurally never indexes depth_logits[-1]; 'all_depths' counts it exactly once more in the auxiliary term. Added exact objective-decomposition telemetry (primary_final_reconstruction_loss, recursive_intermediate_aux_loss, recursive_final_depth_aux_contribution, recursive_depth_aux_weight, combined_training_loss), always populated (explicit 0.0 when disabled, never omitted). resolve_recursive_depth_aux_mode/migrate_recursive_depth_aux_config give any pre-existing config or checkpoint (recursive_depth_aux_mode absent) byte-identical 'legacy_all_depths' behavior -- 'off' is the new default only when recursive_depth_supervision_weights was never set. No production default changed; no adoption decision (deferred to SLM-233)." - }, - { - "version": "v63", - "date": "2026-07-21", - "note": "SLM-237 (RSC-A01): fixed the recursive_depth_supervision_weights training_loss defect -- the loop bound each per-depth weight but never multiplied by it, computing sum(L_d)/sum(w_d) (an unweighted mean) instead of the intended sum(w_d*L_d)/sum(w_d). Added validate_recursive_depth_supervision/ValidatedDepthSupervision: fails closed on unsupported architecture, length mismatch, negative/NaN/inf weights, and all-zero configs (previously all silently accepted/truncated/ignored); empty tuple remains feature-off with no behavior change. Added recursive_depth_weight_{d}/recursive_depth_weighted_contribution_{d}/recursive_depth_supervision_weight_sum/recursive_depth_supervision_enabled telemetry. RSC-A02 (whether the final depth belongs in the auxiliary term) is out of scope." - }, - { - "version": "v62", - "date": "2026-07-20", - "note": "E630: _required_slot_margin_bias now gates on the active argument position's schema, mirroring _schema_role_slot_bias's own accepts_slot check (x-openui-placeholder-flagged on a component frame, or schema-can-reach-a-visible-slot on an object frame) -- fixes E629's traced rico_eval_test_25 frame_depth>=1 over-stuffing regression (one Button absorbing 5 missing slots across all 5 positional arguments, since the bias's old_max+margin floor always wins against schema_value_bias/schema_opaque_bias/schema_enum_close_bias/schema_opaque_close_bias regardless of margin size); default weight (0.0) and E617 contract-gating unchanged. Also found: this same fix reverts E626/E628's previously-reported ood_dashboard_01 margin=2 gain back to control (byte-identical), since that gain was mechanistically the identical over-stuffing behavior and partly an artifact of _is_meaningful_program's literal \"Card([])\" substring check missing an empty-children Card padded with non-content placeholder args; a widened n=19 resweep with the fix applied also surfaces a third, previously invisible regression (smoke_hero_01) -- net binary-gate effect at margin>=1 is now below control (12/19 vs 13/19), not flat. See docs/design iter-e630." - }, - { - "version": "v61", - "date": "2026-07-20", - "note": "E628: _required_slot_margin_bias now excludes frame_depth == 0 (no component/object frame open yet) from its target set, fixing E627's traced margin=6 root hijack -- the bias only fires once decoding is already inside a real component/object frame; default weight (0.0) and E617 contract-gating unchanged" - }, - { - "version": "v60", - "date": "2026-07-20", - "note": "no-bump: E627 adds required_slot_margin_applications/required_slot_margin_choice_changes DecodeStats counters and a bounded required_slot_margin constrained_selection_traces entry (row/position/frame_depth/candidate kinds) purely to observe required_slot_margin_decode_weight's (E626) existing behavior; no bias formula, weight, or default changed" - }, - { - "version": "v60", - "date": "2026-07-20", - "note": "E626: add required_slot_margin_decode_weight, a default-off decode-time margin that floors the best legal candidate filling a still-missing required slot (analogous to semantic_plan_margin_decode_weight for plan families); contract-gated by the E617 slot_contract_constrained_decode guard" - }, - { - "version": "v80", - "date": "2026-07-20", - "note": "E638 rejection: restore v78 behavior after requiring complete slot coverage collapsed Gallery and regressed fidelity, validity, structure, reward, and latency" - }, - { - "version": "v79", - "date": "2026-07-20", - "note": "E638: require complete visible slot-contract coverage before semantic-plan root closure can activate" - }, - { - "version": "v78", - "date": "2026-07-21", - "note": "SLM-212: add default-off constraint_debt_routing_* fields to ModelBuildConfig" - }, - { - "version": "v77", - "date": "2026-07-21", - "note": "SLM-222: add optimizer_name, muon_lr, adamw_lr, weight_decay, muon_momentum, muon_nesterov, and muon_ns_steps fields to ModelBuildConfig" - }, - { - "version": "v76", - "date": "2026-07-21", - "note": "SLM-211: add tie_output_embedding control, deduplicate tied parameters in trainable_parameters/optimizer_parameter_groups, and explicit checkpoint tie migration" - }, - { - "version": "v75", - "date": "2026-07-20", - "note": "E637: count semantic-plan component families from the emitted choice prefix at every nesting depth so legal wrapper descendants satisfy family coverage without duplicate top-level emission" - }, - { - "version": "v74", - "date": "2026-07-20", - "note": "E636: enforce exact component variants in typed arrays, follow public-schema wrappers to missing visible roles, and close legal optional tails after complete slot coverage" - }, - { - "version": "v73", - "date": "2026-07-20", - "note": "E635: combine final pre-content literal routing with property-compatible slot coverage so direct visible slots cannot be floored into non-placeholder component arguments" - }, - { - "version": "v72", - "date": "2026-07-20", - "note": "E634 rejection: restore v70 early pre-content routing after final-boundary enforcement collapsed Auth to TextContent and regressed every aggregate quality metric" - }, - { - "version": "v71", - "date": "2026-07-20", - "note": "E634: split pre-content literal routing from optional opaque penalties and apply it after repeated-slot margins at the final active-property decision boundary" - }, - { - "version": "v70", - "date": "2026-07-20", - "note": "E633 r2 rejection: restore v68 repeated-slot behavior after property-level restriction collapsed Auth to TextContent and regressed every aggregate quality metric" - }, - { - "version": "v69", - "date": "2026-07-20", - "note": "E633 r2: apply repeated-instance distinct-slot margin only at direct placeholder-annotated component properties so it cannot override pre-content operational string routing" - }, - { - "version": "v68", - "date": "2026-07-20", - "note": "E633: when a required operational string is immediately followed by a visible-content property, floor the legal empty literal above visible slots at only that active property boundary" - }, - { - "version": "v67", - "date": "2026-07-20", - "note": "E632 rejection: restore v65 optional-empty-only schema opaque behavior after required non-content string penalties removed Input inventory and regressed every continuous aggregate metric" - }, - { - "version": "v66", - "date": "2026-07-20", - "note": "E632: extend schema_opaque_decode_weight to penalize visible slots in required or optional non-content string arguments while preserving content-annotated properties" - }, - { - "version": "v65", - "date": "2026-07-20", - "note": "E631: when a component owner cannot own any missing visible role, prefer closing it instead of nesting a different role-compatible component at the wrong structural depth" - }, - { - "version": "v64", - "date": "2026-07-20", - "note": "E630 rejection: restore v62 broad schema-role closure behavior after prompt-owned filtering nested Input inside Button and regressed every aggregate quality metric" - }, - { - "version": "v63", - "date": "2026-07-20", - "note": "E630: prefer explicit prompt-owned component candidates over broad schema-only role matches when preventing premature slot-coverage closure" - }, - { - "version": "v62", - "date": "2026-07-20", - "note": "E622: emit bounded slot-coverage closure score traces plus application and immediate-choice-change counters without changing decode scores" - }, - { - "version": "v61", - "date": "2026-07-20", - "note": "E621: require public-schema owner compatibility before a direct visible-slot continuation can outrank frame closure" - }, - { - "version": "v60", - "date": "2026-07-20", - "note": "E621: extend slot_coverage_close_decode_weight to floor a compiler-legal role-compatible component or object-property continuation above premature frame closure while abstaining when compatibility cannot be proven" - }, - { - "version": "v59", - "date": "2026-07-20", - "note": "E617: raise ValueError in _generate_batch_once when schema_role_slot_decode_weight / slot_coverage_close_decode_weight / semantic_plan_typed_array_nonempty_margin_decode_weight / semantic_plan_typed_array_item_margin_decode_weight / semantic_plan_repeated_slot_margin_decode_weight is set without slot_contract_constrained_decode or template_fill_decode, since self._slot_contracts silently stays None otherwise and these biases no-op every step" - }, - { - "version": "v58", - "date": "2026-07-20", - "note": "score visible slots against active inline object properties derived from the public schema" - }, - { - "version": "v57", - "date": "2026-07-20", - "note": "preserve closed object schemas through choice decoding and require declared properties" - }, - { - "version": "v56", - "date": "2026-07-20", - "note": "add SLM-176 P14 action-shortlist mode/config fields, CLI args, and optional retrieve-then-rerank in _select_compiler_path; defaults preserve off behavior" - }, - { - "version": "v55", - "date": "2026-07-20", - "note": "optionally floor the schema-derived item start inside authored typed arrays" - }, - { - "version": "v54", - "date": "2026-07-20", - "note": "optionally start empty slot-bearing typed arrays inside authored prompt-plan components" - }, - { - "version": "v53", - "date": "2026-07-20", - "note": "add SLM-174 SDE2-07 action-alias mode/manifest/name-mode config fields, CLI args, and teacher-init wiring; defaults preserve canonical behavior" - }, - { - "version": "v52", - "date": "2026-07-20", - "note": "optionally floor the best unused visible slot inside each repeated prompt-plan instance" - }, - { - "version": "v51", - "date": "2026-07-20", - "note": "select the deepest repeated semantic-plan ancestor for nested array closure" - }, - { - "version": "v50", - "date": "2026-07-20", - "note": "add default-off repeated-plan-family nested-array close margin and track variadic item counts" - }, - { - "version": "v49", - "date": "2026-07-20", - "note": "add SDE2-03 exposure-targeted rare-action sampling config fields and CLI args; defaults preserve existing behavior" - }, - { - "version": "v48", - "date": "2026-07-20", - "note": "optionally floor verifier-approved semantic-plan root and completion tokens above the best legal score" - }, - { - "version": "v47", - "date": "2026-07-20", - "note": "record bounded score decomposition for verifier-approved semantic-plan root tokens atop default-off SLM-168 pointer plumbing" - }, - { - "version": "v46", - "date": "2026-07-20", - "note": "add SLM-168 pointer_mode / pointer_candidate_source / pointer_* plumbing and standalone dynamic_pointer_scorer module; defaults preserve existing behavior" - }, - { - "version": "v45", - "date": "2026-07-20", - "note": "optionally floor still-required semantic-plan families above the best legal component score" - }, - { - "version": "v44", - "date": "2026-07-20", - "note": "retain all positively biased planned-family candidates outside the diagnostic top-score cutoff" - }, - { - "version": "v43", - "date": "2026-07-20", - "note": "record bounded post-first semantic-plan family score decompositions without changing decode scores" - }, - { - "version": "v42", - "date": "2026-07-20", - "note": "add SLM-166 semantic_connector / train_scope / connector_* plumbing and standalone connector module; defaults preserve existing behavior" - }, - { - "version": "v41", - "date": "2026-07-20", - "note": "extend first-component plan traces through the actual final legal-token selection" - }, - { - "version": "v40", - "date": "2026-07-20", - "note": "record bounded first-component semantic-plan score decompositions in decode evidence" - }, - { - "version": "v39", - "date": "2026-07-20", - "note": "gate first-component prompt-plan scoring on compiler root candidate kinds rather than parser frame emptiness" - }, - { - "version": "v38", - "date": "2026-07-20", - "note": "add a default-off prompt-plan score restricted to the first component-family choice" - }, - { - "version": "v37", - "date": "2026-07-20", - "note": "add SLM-164 legal_margin_mode / targeted_margin_* plumbing; defaults preserve existing behavior" - }, - { - "version": "v36", - "date": "2026-07-20", - "note": "preserve explicit component counts when one descriptive modifier separates the count from the component family" - }, - { - "version": "v35", - "date": "2026-07-20", - "note": "optionally close typed component arrays after all visible slots are covered" - }, - { - "version": "v34", - "date": "2026-07-20", - "note": "add SLM-163 action_embedding_init and action_embedding_train plumbing; defaults preserve existing behavior" - }, - { - "version": "v33", - "date": "2026-07-20", - "note": "no-bump: complete runtime override plumbing for the already registered v33 lever" - }, - { - "version": "v33", - "date": "2026-07-20", - "note": "optionally derive component choices from visible slot roles and the public schema" - }, - { - "version": "v33", - "date": "2026-07-20", - "note": "add SLM-163 action_embedding_init and action_embedding_train plumbing; defaults preserve existing behavior" - }, - { - "version": "v32", - "date": "2026-07-20", - "note": "map visible body and action role aliases to compatible schema properties for prompt-role binding" - }, - { - "version": "v31", - "date": "2026-07-20", - "note": "infer Button in predicted partial plans from authored action and confirmation semantics" - }, - { - "version": "v30", - "date": "2026-07-20", - "note": "score still-missing prompt component families at inline component positions using valid-prefix counts" - }, - { - "version": "v29", - "date": "2026-07-20", - "note": "add default-off legal closure scoring for optional enum-valued component arguments" - }, - { - "version": "v28", - "date": "2026-07-20", - "note": "preserve component-array item schemas in choice decode state so invalid direct placeholder children are compiler-illegal" - }, - { - "version": "v27", - "date": "2026-07-20", - "note": "add default-off visible slot scoring by active content-property owner compatibility" - }, - { - "version": "v26", - "date": "2026-07-20", - "note": "add a default-off legality-preserving score for closing optional unconstrained component arguments" - }, - { - "version": "v25", - "date": "2026-07-20", - "note": "add a default-off legality-preserving bias against visible placeholders in optional unconstrained component arguments" - }, - { - "version": "v24", - "date": "2026-07-20", - "note": "add a default-off legality-preserving schema-value bias that discourages visible placeholders in enum-valued component arguments" - }, - { - "version": "v23", - "date": "2026-07-20", - "note": "bind visible-role gating confidence to complete coverage of the original slot contract so uncovered consumed roles cannot enable it late" - }, - { - "version": "v22", - "date": "2026-07-20", - "note": "abstain from visible-role gating unless every remaining slot has at least one honest role-family candidate" - }, - { - "version": "v21", - "date": "2026-07-20", - "note": "gate the auxiliary learned slot-head bonus to visible role-matching families when honest role evidence is available" - }, - { - "version": "v20", - "date": "2026-07-20", - "note": "derive honest visible slot-to-component candidates from local authored role-family phrases in addition to schema property names" - }, - { - "version": "v19", - "date": "2026-07-20", - "note": "soft-close repeated predicted component instances after their first visible slot so later instances can bind distinct prompt slots" - }, - { - "version": "v18", - "date": "2026-07-20", - "note": "soft-score only still-missing predicted component-family instances so repeated authored requirements can be generated before root closure" - }, - { - "version": "v17", - "date": "2026-07-20", - "note": "preserve repeated authored component mentions in predicted plans and require their family cardinality before verified root closure" - }, - { - "version": "v16", - "date": "2026-07-20", - "note": "soft-follow a compiler- and verifier-valid Stack closure after predicted SemanticPlanV1 component coverage, failing closed without a valid completion" - }, - { - "version": "v15", - "date": "2026-07-20", - "note": "build and terminate a legal Stack root only after predicted SemanticPlanV1 component-family coverage and extend plan binding into its nested child list" - }, - { - "version": "v14", - "date": "2026-07-20", - "note": "compose predicted semantic-plan binding evidence after learned root-reference identity ranking so identity cannot erase the plan factor" - }, - { - "version": "v13", - "date": "2026-07-20", - "note": "soft-rank legal unused terminal-root references whose generated component families match the predicted prompt SemanticPlanV1" - }, - { - "version": "v12", - "date": "2026-07-20", - "note": "consume prompt-derived predicted SemanticPlanV1 component features as legality-preserving choice-codec score bias with intervention telemetry" - }, - { - "version": "v11", - "date": "2026-07-20", - "note": "add SLM-138 shared recursive denoiser tower wiring, deep-supervision plumbing, and migration helper" - }, - { - "version": "v10", - "date": "2026-07-20", - "note": "expose bounded rare slot-owner record sampling in the shared TwoTower training recipe" - }, - { - "version": "v9", - "date": "2026-07-20", - "note": "use a masked non-empty sentinel for terminal slot-pair context so HF tokenization cannot produce a zero-length sequence" - }, - { - "version": "v8", - "date": "2026-07-19", - "note": "smooth slot lexeme-owner priors with class-prior-proportional pseudo-counts so unseen token-class pairs cannot receive spurious positive association" - }, - { - "version": "v7", - "date": "2026-07-19", - "note": "expose the bounded strict-subset root-reference identity sampling multiplier in the shared model-build recipe" - }, - { - "version": "v6", - "date": "2026-07-19", - "note": "add bounded excluded-reference weighting and negative-class accuracy telemetry to terminal-root identity supervision" - }, - { - "version": "v5", - "date": "2026-07-19", - "note": "permute existing terminal-root reference scores by learned identity rank while preserving the reference-group maximum and leaving non-reference candidates unchanged" - }, - { - "version": "v4", - "date": "2026-07-19", - "note": "center terminal-root identity bias over unused legal references so identity ranks references without changing the arity head's continue-versus-stop partition" - }, - { - "version": "v3", - "date": "2026-07-19", - "note": "add bounded terminal-root reference identity supervision and learned legal-reference inclusion bias" - }, - { - "version": "v2", - "date": "2026-07-19", - "note": "bound terminal-root reference-arity targets and auxiliary loss classes by each generated stream's available sections" - }, - { - "version": "v1", - "date": "2026-07-19", - "note": "initial registration; SLM-123 removable low-rank delta adapter backend, factory/config integration, and CLI wiring" - } - ] - }, - "harness.experiments.slm192_profile_flow_pipeline": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm192_profile_flow_pipeline.py", - "tests/test_harnesses/experiments/test_slm192_profile_flow_pipeline.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-192 FFE3-01 stage-accurate cost profiler for valid-edit bridge, training, decode, closure, and verification" - } - ] - }, - "matrix.slm192_profile_flow_pipeline": { - "version": "ffe3-01-v1", - "kind": "matrix", - "paths": [ - "scripts/profile_flow_pipeline.py", - "tests/test_scripts/test_profile_flow_pipeline.py", - "docs/design/iter-slm192-profile-flow-pipeline-20260721.json", - "docs/design/iter-slm192-profile-flow-pipeline-20260721.md" - ], - "history": [ - { - "version": "ffe3-01-v1", - "date": "2026-07-21", - "note": "initial registration; SLM-192 FFE3-01 flow-pipeline cost-profile fixture matrix CLI" - } - ] - }, - "harness.experiments.slm193_flow_caches": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm193_flow_caches.py", - "src/slm_training/harness_core/flow_cache.py", - "tests/test_harnesses/experiments/test_slm193_flow_caches.py", - "tests/test_harness_core/test_flow_cache.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-21", - "note": "initial registration; SLM-193 FFE3-02 bit-exact state, candidate, closure, and bridge cache fixture harness" - } - ] - }, - "matrix.slm193_flow_caches": { - "version": "ffe3-02-v1", - "kind": "matrix", - "paths": [ - "scripts/bench_flow_caches.py", - "tests/test_scripts/test_bench_flow_caches.py", - "docs/design/iter-slm193-flow-caches-20260721.json", - "docs/design/iter-slm193-flow-caches-20260721.md" - ], - "history": [ - { - "version": "ffe3-02-v1", - "date": "2026-07-21", - "note": "no-bump: regenerate SLM-193 design artifact timings after githook fixture run; matrix CLI behavior unchanged" - }, - { - "version": "ffe3-02-v1", - "date": "2026-07-21", - "note": "initial registration; SLM-193 FFE3-02 bit-exact flow-cache fixture matrix CLI" - } - ] - }, - "flow.proposals": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/flow/proposals.py", - "tests/test_flow/test_proposals.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial registration; legality-neutral dynamic candidate proposal prefixes with mandatory exact fallback" - } - ] - }, - "harness.experiments.slm194_candidate_proposals": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm194_candidate_proposals.py", - "tests/test_harnesses/experiments/test_slm194_candidate_proposals.py" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-23", - "note": "initial registration; SLM-194 FFE3-03 candidate proposal recall, fallback, parity, and wall-cost fixture" - } - ] - }, - "harness.experiments.valid_edit_flow_closeout": { - "version": "v3", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/valid_edit_flow_closeout.py", - "tests/test_harnesses/experiments/test_valid_edit_flow_closeout.py" - ], - "history": [ - { - "version": "v3", - "date": "2026-07-24", - "note": "expand the deterministic closeout to the required 18-lever causal table and explicit authorities" - }, - { - "version": "v2", - "date": "2026-07-24", - "note": "stamp the closeout with its dedicated harness and matrix component versions" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; fail-closed SLM-207 evidence lock and causal disposition" - } - ] - }, - "matrix.valid_edit_flow_closeout": { - "version": "vfa3-04-v3", - "kind": "matrix", - "paths": [ - "scripts/run_valid_edit_flow_closeout.py", - "tests/test_scripts/test_run_valid_edit_flow_closeout.py", - "docs/design/valid-edit-flow-closeout.json", - "docs/design/valid-edit-flow-closeout.md", - "docs/design/valid-edit-flow-selected-stack.json", - "docs/design/adr-valid-edit-flow-closeout.md", - "docs/design/valid-edit-flow-architecture.md" - ], - "history": [ - { - "version": "vfa3-04-v3", - "date": "2026-07-24", - "note": "emit ADR, selected-stack inventory, and hard-soft authority map with the complete causal table" - }, - { - "version": "vfa3-04-v2", - "date": "2026-07-24", - "note": "isolate design-output destination so closeout verification tests do not mutate committed evidence" - }, - { - "version": "vfa3-04-v1", - "date": "2026-07-24", - "note": "initial registration; deterministic VFA program closeout" - } - ] - }, - "matrix.slm194_candidate_proposals": { - "version": "ffe3-03-v1", - "kind": "matrix", - "paths": [ - "scripts/run_candidate_proposal_matrix.py", - "tests/test_scripts/test_run_candidate_proposal_matrix.py", - "docs/design/iter-slm194-candidate-proposals-20260724.json", - "docs/design/iter-slm194-candidate-proposals-20260724.md", - "docs/design/iter-slm194-candidate-proposals-agentv-20260724/", - "docs/design/perf-experiment-matrix.md" - ], - "history": [ - { - "version": "ffe3-03-v1", - "date": "2026-07-23", - "note": "no-bump: regenerate measured evidence after final prerequisite squash lineage without changing matrix behavior" - }, - { - "version": "ffe3-03-v1", - "date": "2026-07-23", - "note": "initial registration; SLM-194 FFE3-03 k-grid and true-wall proposal amortization matrix" - } - ] - }, - "harness.eval.replay": { - "version": "v3", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/eval/harness_replay.py", - "scripts/replay_harness_failures.py", - "tests/test_harnesses/eval/test_harness_replay.py", - "docs/design/harness-artifact-audit.md" - ], - "history": [ - { - "version": "v3", - "date": "2026-07-24", - "note": "Run deterministic five-suite local byte-reverification with production scoring, parser cascade, runtime evidence, and automatic evidence-only architecture blocking" - }, - { - "version": "v2", - "date": "2026-07-24", - "note": "Represent missing causal perturbation traces as unknown rather than fabricated zero label flips" - }, - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; byte-preserving archived harness replay records" - } - ] - }, - "harness.experiments.slm310_action_alignment": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm310_action_alignment.py", - "scripts/run_slm310_action_alignment.py", - "tests/test_harnesses/experiments/test_slm310_action_alignment.py", - "docs/design/iter-slm310-action-alignment-20260724.json", - "docs/design/iter-slm310-action-alignment-20260724.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-310 LAR2-03 action alignment: reason-coded proposal telemetry, train-target vs decode-demand distribution audit, declared inverse-action corruption sampler, legacy/corrected STOP-slot accounting arms, matched 2x2 fixture experiment with preregistered thresholds" - } - ] - }, - "harness.experiments.slm312_state_sources": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm312_state_sources.py", - "scripts/run_slm312_state_sources.py", - "tests/test_harnesses/experiments/test_slm312_state_sources.py", - "docs/design/iter-slm312-state-sources-20260724.json", - "docs/design/iter-slm312-state-sources-20260724.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-312 LAR2-04 state sources: offline seedward oracle-guided valid intermediates + immutable content-addressed on-policy beam-trajectory snapshots labeled with the SLM-308 distance oracle and SLM-305 legal edits, fail-closed AST-fingerprint leak guards, matched {gold_only, seedward, on_policy, mixed} mixture arms with preregistered thresholds" - } - ] - }, - "harness.experiments.slm314_winner_take_all": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm314_winner_take_all.py", - "scripts/run_slm314_winner_take_all.py", - "tests/test_harnesses/experiments/test_slm314_winner_take_all.py", - "src/slm_training/resources/data/slm314_multimode/modes.jsonl", - "src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json", - "docs/design/iter-slm314-winner-take-all-20260724.json", - "docs/design/iter-slm314-winner-take-all-20260724.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-314 LAR2-05 winner-take-all (MCL): frozen 8-prompt x 2-mode verifier-accepted canonical-AST dataset (alpha-invariant fingerprints + sha256 manifest), min-loss-mode loss with preregistered 0.1 coverage/floor term, deterministic two-mode synthetic collapse-vs-retain proof, matched {single_gold, multi_gold, wta} arms" - } - ] - }, - "harness.experiments.slm317_repair_hybrid": { - "version": "v1", - "kind": "harness", - "paths": [ - "src/slm_training/harnesses/experiments/slm317_repair_hybrid.py", - "scripts/run_slm317_repair_hybrid.py", - "tests/test_harnesses/experiments/test_slm317_repair_hybrid.py", - "docs/design/iter-slm317-repair-hybrid-20260724.json", - "docs/design/iter-slm317-repair-hybrid-20260724.md" - ], - "history": [ - { - "version": "v1", - "date": "2026-07-24", - "note": "initial registration; SLM-317 LAR2-06 do-no-harm AR->repair hybrid: metamorphism-invariant commit rule (hard ladder + calibrated soft score), matched arms incl. historical/improved repair + oracle upper bound, invalid-over-valid visibility, advancement gates and repair_positive|repair_negative|inconclusive disposition" - } - ] + "schema": "version_registry/v1", + "_readme": [ + "Canonical component-version registry for the self-improving eval/smoke/checkpoint stack.", + "Contract: docs/design/version-stamp-contract.md. Enforced by scripts/verify_version_stamps.py.", + "Any change to a file under a component's `paths` must touch that component's entry in the", + "same change: bump `version` (new history entry, newest first) or append a same-version", + "history entry whose note starts with 'no-bump:' explaining why the change is behavior-neutral.", + "`paths` are repo-relative; a trailing '/' claims a directory prefix; longest prefix wins.", + "Staleness ordering is the position in `history` (newest first), never string comparison.", + "New ids and future bumps use monotonic v1, v2, ...; pre-existing native encodings are kept." + ], + "components": { + "analysis.arity.diffusion_graph": { + "version": "v1", + "kind": "metric", + "paths": [ + "src/slm_training/dsl/analysis/arity/diffusion_graph.py", + "scripts/run_quotient_diffusion_fixture.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration; wiring-first quotient-state diffusion graph diagnostics" } + ] + }, + "config.levers": { + "version": "v20", + "kind": "harness", + "paths": [ + "src/slm_training/levers.py", + "tests/test_levers.py" + ], + "history": [ + { + "version": "v20", + "date": "2026-07-23", + "note": "Narrow the canonical Vercel evidence bundle to top-level design JSON consumed by the deployed observability reader" + }, + { + "version": "v19", + "date": "2026-07-23", + "note": "Add fail-closed CAP0, CAP1, and CAP2 lever profiles with separate teacher and distillation permission" + }, + { + "version": "v18", + "date": "2026-07-23", + "note": "Raise the canonical run cap to the permitted three minutes so changed-test CI can complete after dependency setup" + }, + { + "version": "v17", + "date": "2026-07-22", + "note": "Generate Vercel duration and committed-evidence bundle inputs from the canonical run-policy levers" + }, + { + "version": "v16", + "date": "2026-07-22", + "note": "Move changed-test parallelism into the canonical run-policy module and expose it through the single lever catalog" + }, + { + "version": "v15", + "date": "2026-07-22", + "note": "Require visible slot inventory whenever slot-aware or semantic-role decode levers are enabled" + }, + { + "version": "v14", + "date": "2026-07-22", + "note": "Declare required-slot margin dual-path after wiring the shared bias into lexer compiler selection" + }, + { + "version": "v13", + "date": "2026-07-22", + "note": "Expose the existing slot-coverage close lever on lexer restricted and tree compiler paths" + }, + { + "version": "v12", + "date": "2026-07-22", + "note": "Centralize decode-lever companion dependencies and fail before artifacts when required slot-contract or visible semantic-role configuration is absent" + }, + { + "version": "v11", + "date": "2026-07-22", + "note": "Expose active-schema visible-slot role binding on lexer compiler paths" + }, + { + "version": "v10", + "date": "2026-07-22", + "note": "Expose the existing semantic-plan typed-array nonempty margin on lexer compiler paths" + }, + { + "version": "v9", + "date": "2026-07-22", + "note": "Expose semantic-plan family scoring and margin as executable lexer compiler levers" + }, + { + "version": "v8", + "date": "2026-07-22", + "note": "Withdraw unreachable lexer root-reference identity supervision and decoding so invalid configurations fail before artifacts" + }, + { + "version": "v7", + "date": "2026-07-22", + "note": "Expose trained root-reference identity supervision and decoding on the lexer compiler path" + }, + { + "version": "v6", + "date": "2026-07-22", + "note": "Declare trained-objective dependencies for learned decode heads and reject untrained or inert configurations before execution" + }, + { + "version": "v5", + "date": "2026-07-22", + "note": "Expose root-reference arity as an executable lexer compiler lever instead of rejecting the symbol-only path" + }, + { + "version": "v4", + "date": "2026-07-21", + "note": "Expose the atomic evaluation policy preset in the canonical lever catalog" + }, + { + "version": "v3", + "date": "2026-07-22", + "note": "Unify decode-lever capability requirements and reject configurations with no executable choice or compiler path before runs start" + }, + { + "version": "v2", + "date": "2026-07-22", + "note": "Declare output-tokenizer applicability in the canonical lever catalog and fail closed on incompatible enabled levers" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "centralize run policy and expose the complete ModelBuildConfig lever catalog with context-specific defaults" + } + ] + }, + "data.corrupt.trace": { + "version": "v1", + "kind": "data_builder", + "paths": [ + "src/slm_training/data/corrupt/trace.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration; CorruptionTraceV2 severity taxonomy and fail-closed validation for SLM-120" + } + ] + }, + "data.flow.bridge_planner": { + "version": "v2", + "kind": "data_builder", + "paths": [ + "src/slm_training/data/flow/bridge_planner.py", + "tests/test_data/flow/test_bridge_planner.py" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "make every bridge transition certificate independently replayable and replace unconditional verifier acceptance with canonical validation" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-189 FFE2-01 bridge planner protocols and deterministic engine wiring/fixture" + } + ] + }, + "data.flow.legal_edit_bridge_corpus": { + "version": "v2", + "kind": "data_builder", + "paths": [ + "src/slm_training/data/flow/bridge_corpus.py", + "scripts/build_legal_edit_bridges.py", + "tests/test_data/flow/test_bridge_corpus.py", + "tests/test_scripts/test_build_legal_edit_bridges.py", + "tests/fixtures/slm196_legal_edit_bridge/", + "src/slm_training/resources/data/train/slm196_legal_edit_bridge_fixture/", + "docs/design/legal-edit-bridge-corpus.md", + "docs/design/iter-slm196-legal-edit-bridge-20260723.json", + "docs/design/iter-slm196-legal-edit-bridge-20260723.md" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "remove planner-selected edit focus and dependency fields from the model-facing row view while retaining them only as diagnostics" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial registration; SLM-196 replay-exact full-bridge rows, content-addressed live candidate sets, quality gates, and fixture builder" + } + ] + }, + "data.semantic_contrast": { + "version": "v1", + "kind": "data_builder", + "paths": [ + "src/slm_training/data/semantic_contrast/", + "scripts/build_semantic_contrasts.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SPV2-01 versioned hard-valid semantic-contrast corpus and corruption taxonomy" + } + ] + }, + "data.test_build": { + "version": "v6", + "kind": "data_builder", + "paths": [ + "scripts/build_test_data.py", + "src/slm_training/harnesses/test_data/", + "src/slm_training/resources/test_seeds.jsonl" + ], + "history": [ + { + "version": "v6", + "date": "2026-07-23", + "note": "no-bump: clarify that the preceding v6 entry was misattributed registry metadata from SLM-230 and made no test-data builder behavior change" + }, + { + "version": "v6", + "date": "2026-07-23", + "note": "pin committed evidence stamps during reproduction and exclude elapsed runtime noise from the scientific report hash" + }, + { + "version": "v5", + "date": "2026-07-22", + "note": "Make canonical held-out targets symbol-only and structurally disjoint without weakening leakage gates" + }, + { + "version": "v4", + "date": "2026-07-22", + "note": "Stamp test-data build evidence and resolve train-manifest records from the owning checkout" + }, + { + "version": "v3", + "date": "2026-07-22", + "note": "Assert the symbol-only output contract after test-target normalization and fail before artifacts when sanitization is disabled or falls back" + }, + { + "version": "v2", + "date": "2026-07-19", + "note": "apply the shared deterministic target sanitization (enforce by default) so future eval gold matches the sanitized train distribution; committed snapshots stay immutable" + }, + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration (eval/smoke suite builder; snapshot data versions remain DataStore's contract)" + } + ] + }, + "dsl.action_descriptions": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/action_descriptions.py", + "tests/test_dsl/test_action_descriptions.py" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-20", + "note": "add fixture_vectors helper for deterministic description vector generation used by SLM-176 shortlist wiring" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-174 SDE2-07 action-alias map and alias-aware description sources" + } + ] + }, + "dsl.action_shortlist": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/action_shortlist.py", + "tests/test_dsl/test_action_shortlist.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-176 P14 description-based retrieve-then-rerank policy and wiring helpers" + } + ] + }, + "dsl.operators.contracts": { + "version": "v12", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/", + "tests/test_dsl/test_operator_contracts.py" + ], + "history": [ + { + "version": "v12", + "date": "2026-07-23", + "note": "add the default-off versioned reserved operator target codec, live legal-set application boundary, and checkpoint compatibility contract" + }, + { + "version": "v11", + "date": "2026-07-23", + "note": "export replay-authoritative ordered conversation collapse decisions and reordered hard-negative evidence" + }, + { + "version": "v10", + "date": "2026-07-23", + "note": "persist bounded typed rejection applications with exact legal-set evidence for symbolic operator corpus generation" + }, + { + "version": "v9", + "date": "2026-07-23", + "note": "export conservative replayable branch merge decisions with exact structural composition and typed conflicts" + }, + { + "version": "v8", + "date": "2026-07-23", + "note": "export immutable replayable conversation state graphs, explicit history operations, and branch-local reference cloning" + }, + { + "version": "v7", + "date": "2026-07-23", + "note": "export semantic-first equivalent-outcome preference costs, sequence diagnostics, and bounded deterministic preference groups" + }, + { + "version": "v6", + "date": "2026-07-23", + "note": "export bounded exact operator legal-set enumeration, reserved typed serialization, and conservative forcing APIs" + }, + { + "version": "v5", + "date": "2026-07-23", + "note": "export the exact topology and template-alias operator context and composed library APIs" + }, + { + "version": "v4", + "date": "2026-07-23", + "note": "export the exact OpenUI core local operator family and preserve stable executor rejection codes" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "export state-bound permutation-invariant reference table and descriptor APIs" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "validate bound operator arguments against required declaration slots and reference kinds before pack execution" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial compiler-owned AST operator, typed reference, action-effect, and application proof contracts" + } + ] + }, + "dsl.operators.conversation": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/conversation.py", + "tests/test_dsl/test_operator_conversation.py" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "test exact multi-turn collapse, history refusal, cycle refusal, and equivalent reorder preservation" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial immutable conversation state DAG with exact operator replay, undo, redo, checkout, fork, and branch-local reference invalidation" + } + ] + }, + "dsl.operators.collapse": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/collapse.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial exact ordered trace collapse with typed refusal and verified reordered hard negatives" + } + ] + }, + "dsl.grammar_capabilities": { + "version": "v3", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/grammar_capabilities.py", + "tests/test_dsl/test_grammar_capabilities.py" + ], + "history": [ + { + "version": "v3", + "date": "2026-07-24", + "note": "declare typed pack-owned finite completion domains with scope fingerprints and terminal witnesses" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "add exact Lark production occurrence traces and pack-owned witness candidate and unsupported authorities" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial pack-owned exact grammar capability adapter with typed unsupported results, analyses, fingerprints, and shared conformance" + } + ] + }, + "dsl.harness_dsl": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/harness_dsl.py", + "src/slm_training/dsl/grammars/harness.lark", + "tests/test_dsl/test_harness_dsl.py", + "docs/design/dsh1-03-symbolic-harness-dsl-20260723.json", + "docs/design/dsh1-03-symbolic-harness-dsl-20260723.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial closed versioned Harness DSL with reserved CAP0 operations, typed payloads, pack validation, declared markers, artifact refs, and canonical fingerprinted framing" + } + ] + }, + "dsl.minimal_witnesses": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/minimal_witnesses.py", + "tests/test_dsl/test_minimal_witnesses.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial per-start reachable productive alternative basis with lexicographic witnesses, exact focus traces, typed gaps, and fail-closed admission" + } + ] + }, + "dsl.operators.merge": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/merge.py", + "tests/test_dsl/test_operator_merge.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial bounded three-way branch merge with fork-ref lineage, pack revalidation, deterministic replay, and typed conflict artifacts" + } + ] + }, + "dsl.operators.references": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/references.py", + "tests/test_dsl/test_operator_references.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial state-bound opaque typed reference descriptors, stable resolution failures, ordered-parent index guards, and permutation controls" + } + ] + }, + "dsl.operators.registry": { + "version": "v3", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/registry.py", + "src/slm_training/dsl/pack.py", + "tests/test_dsl/test_operator_registry.py" + ], + "history": [ + { + "version": "v3", + "date": "2026-07-24", + "note": "bind OpenUI scoped completion domains to the pack contract instead of shared decoder policy" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: populate the existing fragment-parser slot for OpenUI without changing operator registry behavior" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: add pack-owned witness candidates and unsupported alternative reasons without changing operator registry behavior" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: add an independent grammar capability authority slot without changing operator registry behavior" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "preserve stable executor rejection codes and failed preconditions through pack-owned application evidence" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial pack-owned immutable operator registry with pure apply, dry-run authority validation, typed rejection, and exact replay" + } + ] + }, + "dsl.operators.local": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/local.py", + "tests/test_dsl/test_local_operators.py" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "expose core registered entries for composition into the pack-owned topology library" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial exact OpenUI add, remove, replace, property, and child-order operator family" + } + ] + }, + "dsl.operators.legal_set": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/legal_set.py", + "tests/test_dsl/test_operator_legal_set.py" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "retain bounded deterministic rejected application samples alongside complete rejection counts" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial exact bounded operator-action enumeration with complete/partial coverage, brute-force shadow controls, and reserved hierarchical serialization" + } + ] + }, + "dsl.operators.reserved_tokens": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/reserved_tokens.py", + "tests/test_dsl/test_reserved_operator_tokens.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial default-off v1 reserved target framing with closed parsing, exact live legal-set membership, compiler application, typed defer/reject reasons, and checkpoint incompatibility guards" + } + ] + }, + "dsl.operators.preference": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/preference.py", + "tests/test_dsl/test_operator_preference.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial semantic-gated lexicographic AST and operator-sequence preference contract with simplify/expand direction and explicit defect policy" + } + ] + }, + "dsl.operators.topology": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/operators/topology.py", + "tests/test_dsl/test_topology_operators.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial exact move, reparent, wrap, unwrap, duplicate, expand, and contract operator family" + } + ] + }, + "dsl.scope_env": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/scope_env.py", + "tests/test_dsl/test_scope_env.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; Torch-free persistent typed ScopeEnv with opaque ordinal IDs, separate surface aliases, and explicit shadowing/forward-reference policies" + } + ] + }, + "dsl.symbolic_surface": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/language_contract.py", + "tests/test_dsl/test_language_contract.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial pack-authoritative SymbolicSurfacePolicyV1 with declared runtime markers, typed spans and decisions, and alpha/opaque-ID permutation controls" + } + ] + }, + "dsl.solver.topology": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/dsl/solver/topology_adapter.py", + "src/slm_training/dsl/solver/topology_solver.py", + "tests/test_dsl/test_topology_adapter.py", + "tests/test_dsl/test_topology_solver.py", + "docs/design/verified-scope-solver.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "no-bump: referenced by SLM-188 edit-algebra harness; topology edit application unchanged" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; VSS3-03 topology finite-domain adapter/solver seam (referenced by SLM-187 parity fixture; no behavior change in this commit)" + } + ] + }, + "evals.agentv": { + "version": "v4", + "kind": "harness", + "paths": [ + "src/slm_training/evals/agentv.py", + "scripts/run_agentv_eval.mjs", + "tests/test_evals/test_agentv.py" + ], + "history": [ + { + "version": "v4", + "date": "2026-07-24", + "note": "Replace the optional Node LangSmith SDK with bounded native fetch so Vercel does not bundle CLI-only dependencies" + }, + { + "version": "v3", + "date": "2026-07-24", + "note": "Forward valid W3C run traces to the pinned AgentV runner for summary-only observational telemetry" + }, + { + "version": "v2", + "date": "2026-07-24", + "note": "Make raw AgentEvals criteria code-graded authority and derive verdicts from AgentV's published artifact index" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial registration; independently enforce named case checks and attach canonical version stamps to AgentV result artifacts" + } + ] + }, + "evals.cap2_operator": { + "version": "v1", + "kind": "metric", + "paths": [ + "src/slm_training/evals/cap2_operator.py", + "src/slm_training/resources/evals/cap2_operator_v1.json", + "scripts/run_cap2_operator_eval_fixture.py", + "tests/test_evals/test_cap2_operator.py", + "tests/test_scripts/test_run_cap2_operator_eval_fixture.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: document opt-in LangSmith telemetry configuration in the shared README without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial frozen CAP2 symbolic operator suite with replayed held-out composition, confidence bounds, telemetry, and anti-cheat controls" + } + ] + }, + "evals.cap2_disposition": { + "version": "v1", + "kind": "metric", + "paths": [ + "src/slm_training/evals/cap2_disposition.py", + "scripts/publish_cap2_disposition.py", + "tests/test_evals/test_cap2_disposition.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial typed terminal CAP2 capability ledger with exact evidence identities, CERT_CAP2 rejection, and DSH4 action-distillation closure" + } + ] + }, + "evals.judge_resolution": { + "version": "v1", + "kind": "metric", + "paths": [ + "src/slm_training/evals/judge_resolution.py", + "scripts/run_judge_resolution_audit.py", + "tests/test_evals/test_judge_resolution.py", + "tests/test_scripts/test_run_judge_resolution_audit.py", + "docs/design/iter-slm185-judge-resolution-20260720.json", + "docs/design/iter-slm185-judge-resolution-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-185 FFE0-03 judge test-retest reliability, canonical-equivalence invariance, and semantic resolution floor wiring/fixture harness" + } + ] + }, + "evals.loss_suite": { + "version": "v1", + "kind": "metric", + "paths": [ + "src/slm_training/evals/loss_suites.py", + "src/slm_training/evals/loss_suite_v1.json", + "src/slm_training/evals/denoising_nll.py", + "src/slm_training/evals/emptiness_probe.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-19", + "note": "no-bump: extend emptiness probe with optional eval-only score-policy comparison; LOSS_SUITE_VERSION unchanged" + }, + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration; mirrors loss_suites.LOSS_SUITE_VERSION" + } + ] + }, + "evals.meaningful_program": { + "version": "2.13.0", + "kind": "metric", + "paths": [ + "src/slm_training/evals/meaningful_program.py" + ], + "history": [ + { + "version": "2.13.0", + "date": "2026-07-22", + "note": "Reject duplicate declared template-marker identities instead of treating set coverage as sufficient" + }, + { + "version": "2.12.0", + "date": "2026-07-21", + "note": "Reject every meaningfulness candidate containing a free-form output string under the symbol-only output contract" + }, + { + "version": "2.11.0", + "date": "2026-07-21", + "note": "E713: apply mechanical text-only routing rejection only when an explicitly requested component can directly own a required semantic role" + }, + { + "version": "2.10.0", + "date": "2026-07-21", + "note": "E712: preserve explicit component counts across arbitrary descriptive modifiers while stopping at conjunction, preposition, or component boundaries" + }, + { + "version": "2.9.0", + "date": "2026-07-21", + "note": "E699 rejection: restore pre-alias role semantics after generic placeholder/value aliases collapsed unrelated email and submit roles into RadioItem" + }, + { + "version": "2.8.0", + "date": "2026-07-21", + "note": "E699: align form-role schema aliases with the metric's accepted Input, TextArea, and Select placeholder/value properties" + }, + { + "version": "2.7.0", + "date": "2026-07-21", + "note": "E698: treat component-like adjectives as descriptive prose rather than explicit component requirements" + }, + { + "version": "2.6.0", + "date": "2026-07-21", + "note": "E685: normalize numbered visible roles to their base and align canonical tab triggers plus overview display text with public-schema properties" + }, + { + "version": "2.5.0", + "date": "2026-07-21", + "note": "E684: recognize schema plural component families through singular prose forms while preserving one container for singular compound mentions" + }, + { + "version": "2.4.0", + "date": "2026-07-21", + "note": "E680: recognize kicker and heading as schema-valid display-text roles so visible prose can occupy TextContent.text" + }, + { + "version": "2.3.0", + "date": "2026-07-21", + "note": "E674: accept public-schema aliases used by canonical fixtures: action may occupy Button.label, body may occupy description, and display value may occupy TextContent.text" + }, + { + "version": "2.2.1", + "date": "2026-07-20", + "note": "E636 r1 correction: preserve the established form-slot compatibility contract for Input, TextArea, and Select placeholder/value properties while retaining explicit rejection of Input.name" + }, + { + "version": "2.2.0", + "date": "2026-07-20", + "note": "E636: validate recognized placeholder roles against their public-schema string property instead of hard-coded component families, allowing schema-declared roles such as Modal.title while retaining Input.name rejection" + }, + { + "version": "2.1.0", + "date": "2026-07-20", + "note": "E618: _binding_check no longer falls back to the regex-based Gate.REFERENCES gate for structural (non-runtime-syntax) sources -- that fallback treated bare object-literal property keys (e.g. `src:` in a typed-array item `{src: ..., alt: ...}`) as unresolved variable references, permanently failing binding_correctness (reason `reference_graph_invalid`) for any correctly produced typed-array-of-objects prediction. The official-parser-derived unresolved/orphaned checks plus the $state/Query/Mutation dependency-graph reachability pass (already used for runtime sources, a safe no-op for structural sources) now run unconditionally instead." + }, + { + "version": "2.0.0", + "date": "2026-07-18", + "note": "initial registration; mirrors meaningful_program.METRIC_VERSION" + } + ] + }, + "evals.power_protocol": { + "version": "v7", + "kind": "metric", + "paths": [ + "src/slm_training/evals/power_protocol.py", + "tests/test_evals/test_power_protocol.py" + ], + "history": [ + { + "version": "v7", + "date": "2026-07-23", + "note": "Add exact paired-binary inference for preregistered shared-draw control and candidate arms" + }, + { + "version": "v6", + "date": "2026-07-23", + "note": "Add deterministic Holm step-down correction for prospectively declared hypothesis families" + }, + { + "version": "v5", + "date": "2026-07-23", + "note": "Reject boolean and floating-point preregistration sidedness values" + }, + { + "version": "v4", + "date": "2026-07-23", + "note": "Reject duplicate, boolean, and non-integer preregistration seed identifiers" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "Add prospective one-proportion sample-size planning with separate per-seed analyses and no observed-power inputs" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "Add configurable fail-closed Wilson intervals and canonical binomial rate-evidence metadata" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: ruff style fixes (rename ambiguous lambda variables); behavior-neutral" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-183 PQR powered cluster-aware confirmation protocol statistical utilities" + } + ] + }, + "evals.render_equivalence": { + "version": "v1", + "kind": "metric", + "paths": [ + "src/slm_training/evals/render_equivalence.py", + "tests/test_evals/test_render_equivalence.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-172 SDE2-05 tiered canonical/render-tree/visual-diff render-equivalence surrogate" + } + ] + }, + "evals.scoring": { + "version": "v22", + "kind": "metric", + "paths": [ + "src/slm_training/evals/" + ], + "history": [ + { + "version": "v22", + "date": "2026-07-23", + "note": "Add the terminal CAP2 capability disposition and fail-closed certificate decision" + }, + { + "version": "v21", + "date": "2026-07-23", + "note": "Add the frozen CAP2 operator, history, collapse, permutation, and anti-cheat scoring contract" + }, + { + "version": "v20", + "date": "2026-07-23", + "note": "Add exact paired-binary inference for governed campaign endpoint evidence" + }, + { + "version": "v19", + "date": "2026-07-23", + "note": "Add deterministic Holm family-wise correction with complete per-hypothesis evidence" + }, + { + "version": "v18", + "date": "2026-07-23", + "note": "Fail closed on non-integer prospective power sidedness" + }, + { + "version": "v17", + "date": "2026-07-23", + "note": "Fail closed on malformed prospective power seed identifiers" + }, + { + "version": "v16", + "date": "2026-07-23", + "note": "Add prospective binomial power preregistration metadata without post-hoc observed-power claims" + }, + { + "version": "v15", + "date": "2026-07-23", + "note": "Add count-provenanced configurable Wilson evidence for canonical evaluator binomial rates" + }, + { + "version": "v14", + "date": "2026-07-22", + "note": "Templatize positive SwitchItem and TabItem metric-gaming archetypes so accepted fixtures contain no free-form output strings" + }, + { + "version": "v13", + "date": "2026-07-21", + "note": "add OracleScoringReplayV1 harness and audit_gold_scoring CLI; existing scoring helpers unchanged" + }, + { + "version": "v12", + "date": "2026-07-21", + "note": "no-bump: SLM-186 adds new Goodhart canary slices to metric_gaming.py; existing slice scoring unchanged" + }, + { + "version": "v12", + "date": "2026-07-20", + "note": "wire default-off render_equivalence surrogate into task_scoreboard; RENDER_EQUIVALENCE_MODE remains off" + }, + { + "version": "v11", + "date": "2026-07-20", + "note": "add SPV2-04 dense teacher distribution mixture harness (dense_teacher_mixture.py) and fixture runner on top of the EFS3-01 state-source owner and SPV2-03 legal-set teacher traces" + }, + { + "version": "v10", + "date": "2026-07-19", + "note": "no-bump: relocate record_schema/eval_cache/score_policy to harness_core (old modules are sys.modules alias shims); suite_sharding and all scoring behavior unchanged" + }, + { + "version": "v10", + "date": "2026-07-19", + "note": "add EFS0-04 judge-independence audit scaffolding (JudgeIndependenceEvaluator, judge_audit annotation harness, export_annotations CLI) and rubric" + }, + { + "version": "v9", + "date": "2026-07-19", + "note": "add EFS2-01 X22 tree-edit beam-width \u00d7 edit-depth scaling harness (TreeEditScalingConfig, run_scaling_grid, valid-state beam search) on top of quantized-energy/solver-state/verifier-cascade stack" + }, + { + "version": "v8", + "date": "2026-07-19", + "note": "add CAP4-03 quantized local-energy inference comparison harness (EnergyProblem, EnergyQuantizer, greedy vs exact Viterbi, QuantFormat wiring) on top of solver-state/verifier-cascade stack" + }, + { + "version": "v7", + "date": "2026-07-19", + "note": "add EFS3-01 solver-state supervision source comparison schema and mixer on top of verifier-cascade/score-policy/eval-cache stack" + }, + { + "version": "v6", + "date": "2026-07-19", + "note": "add cached cheap-to-expensive verifier cascade scaffolding (VerifierStageSpec, VerifierResultV1, VerifierCascade) on top of score-policy/eval-cache/trigger-telemetry stack" + }, + { + "version": "v5", + "date": "2026-07-19", + "note": "add EFS1-03 eval-only score-policy interface for constrained-decode candidate ranking on top of eval cache/trigger telemetry/metric-gaming stack" + }, + { + "version": "v4", + "date": "2026-07-19", + "note": "merge SDE3-01 eval cache and sharding on top of EFS2-02 trigger telemetry and SDE0-02 metric-gaming suite" + }, + { + "version": "v3", + "date": "2026-07-19", + "note": "add SDE3-01 content-addressed eval cache (EvalCache, EvalCacheKey, layer keys) and deterministic suite sharding (assign_example_ids, aggregate_shard_payloads)" + }, + { + "version": "v3", + "date": "2026-07-19", + "note": "merge EFS2-02 trigger telemetry harness on top of SDE0-02 metric-gaming stress suite" + }, + { + "version": "v2", + "date": "2026-07-19", + "note": "add EFS2-02 observe-only trigger telemetry harness (SearchTriggerObservationV1, TriggerObserver, compare_trigger_regimes) and wiring fixture" + }, + { + "version": "v2", + "date": "2026-07-19", + "note": "add SDE0-02 metric-gaming stress suite (metric_gaming.py) with deterministic adversarial cases scored by binding_aware_meaningful_v2" + }, + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration (catch-all for eval scoring helpers not claimed by a narrower component)" + } + ] + }, + "evals.semantic_bits": { + "version": "v1", + "kind": "metric", + "paths": [ + "src/slm_training/evals/semantic_bits.py", + "tests/test_evals/test_semantic_bits.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-186 compiler_state_conditional_bits and verified-utility efficiency helpers" + } + ] + }, + "evals.verified_utility": { + "version": "v1", + "kind": "metric", + "paths": [ + "src/slm_training/evals/verified_utility.py", + "tests/test_evals/test_verified_utility.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-186 VerifiedUtilityV1 schema, scalar/lexicographic ranking, pareto, CVaR, abstention economics, sensitivity, and canonical-equivalence helpers" + } + ] + }, + "flow.reference": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/flow/reference/" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "add SLM-190 FFE2-02 exact finite-state CTMC reference package (adapter, enumeration, generator, sampler, lumpability, trajectory, FlowTargetRowV1) and torch-free adapters" + } + ] + }, + "flow.termination": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/flow/termination.py", + "tests/test_flow/termination/test_termination_policy.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-191 FFE2-03 shared TerminationPolicy protocol, arms, and exact-CTMC sampler instrumentation" + } + ] + }, + "features.openfeature": { + "version": "v9", + "kind": "harness", + "paths": [ + "src/slm_training/features/keys.py", + "src/slm_training/features/defaults.py", + "src/slm_training/features/levers.py", + "src/slm_training/features/runtime.py", + "src/slm_training/web/observability.py", + "src/slm_training/web/app.py", + "src/slm_training/web/routes.py", + "src/apps/dashboard/src/features/keys.ts", + "src/apps/dashboard/src/features/runtime.ts", + "src/apps/dashboard/src/interpret/library.tsx", + "src/apps/dashboard/src/interpret/toolProvider.ts", + "src/apps/dashboard/src/main.tsx", + "src/apps/dashboard/src/pages/Experiments.tsx", + "src/apps/dashboard/src/pages/ExperimentFlagDetail.tsx", + "docs/design/openfeature-experiments.md" + ], + "history": [ + { + "version": "v9", + "date": "2026-07-24", + "note": "Expose AgentEvals authoritative criterion summaries beside AgentV runner status on experiment scoreboards" + }, + { + "version": "v8", + "date": "2026-07-24", + "note": "Link committed model-card parameters and checkpoint evidence to dashboard scoreboards and roster metrics" + }, + { + "version": "v7", + "date": "2026-07-24", + "note": "Expose the isolated research-harness OpenFeature capability and OFREP read endpoints beside product flags" + }, + { + "version": "v6", + "date": "2026-07-23", + "note": "no-bump: route the promotion endpoint through the frozen core so the Vercel bundle excludes offline experiment runners without changing the API contract" + }, + { + "version": "v6", + "date": "2026-07-23", + "note": "Add OpenFeature flag detail evidence, source excerpts, and outcome comparisons to the experiment dashboard" + }, + { + "version": "v5", + "date": "2026-07-23", + "note": "Use canonical OpenFeature keys in product feature metadata and bootstrap payloads" + }, + { + "version": "v4", + "date": "2026-07-23", + "note": "Expose the generated model-build OpenFeature lever registry through the experiment dashboard API" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "Accept and fail-closed validate campaign governance evidence in the promotion evaluation API" + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "no-bump: doc-only cross-link to openfeature-autoresearch-experiments.md and terminology-table row for the autoresearch flagd representation; runtime, keys, defaults, and lever registry unchanged" + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "LaunchDarkly OpenFeature provider, experiment lever registry, /api/features/levers, server-side LD browser model" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "initial OpenFeature product experiment layer: in-memory + optional PostHog providers, /api/features/bootstrap, dashboard default-renderer flag" + } + ] + }, + "gates.ship": { + "version": "openui_ship_gates_v4", + "kind": "gate", + "paths": [ + "src/slm_training/harnesses/model_build/ship_gates.py", + "src/slm_training/harness_core/gate_engine.py" + ], + "history": [ + { + "version": "openui_ship_gates_v4", + "date": "2026-07-24", + "note": "Fail closed on audited X22 suite reachability below 1.0 as a measurement-integrity failure" + }, + { + "version": "openui_ship_gates_v3", + "date": "2026-07-24", + "note": "Make raw AgentEvals assertions the canonical ship-gate authority while retaining strict integrity and runtime policy" + }, + { + "version": "openui_ship_gates_v2", + "date": "2026-07-23", + "note": "no-bump: mirror the already-versioned v2 runtime and integrity policy in the meaningful-metric provenance descriptor" + }, + { + "version": "openui_ship_gates_v2", + "date": "2026-07-23", + "note": "Fail closed on malformed measurement values and explicit decode timeouts while preserving evidence and quality categories" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-23", + "note": "no-bump: add failure-category diagnostics without changing thresholds, gate checks, or pass decisions" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-22", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.12.0 to 2.13.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.11.0 to 2.12.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.10.0 to 2.11.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.9.0 to 2.10.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.8.0 to 2.9.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.7.0 to 2.8.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.6.0 to 2.7.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.5.0 to 2.6.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.4.0 to 2.5.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.3.0 to 2.4.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align the ungated binding_aware_meaningful_v2 provenance descriptor from 2.2.1 to 2.3.0; thresholds remain disabled and ship policy is unchanged" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-21", + "note": "no-bump: align MEANINGFUL_METRIC_POLICY binding_aware_meaningful_v2 provenance descriptor version from 2.1.0 to 2.2.1 to match evals.meaningful_program/METRIC_VERSION; thresholds remain None (candidate_pending_calibration), no gate relaxation" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-20", + "note": "no-bump: align MEANINGFUL_METRIC_POLICY binding_aware_meaningful_v2 provenance descriptor version from 2.0.0 to 2.1.0 to match evals.meaningful_program/METRIC_VERSION; thresholds remain None (candidate_pending_calibration), no gate relaxation" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-19", + "note": "no-bump: relocate the generic gate-check loop to harness_core.gate_engine behind a normalize_suite/default_min_n seam; policy, thresholds, and the gates.json payload stay byte-identical (pinned by tests/test_harness_core/test_gate_engine_golden.py)" + }, + { + "version": "openui_ship_gates_v1", + "date": "2026-07-18", + "note": "initial registration; mirrors MEANINGFUL_METRIC_POLICY threshold_version" + } + ] + }, + "harness.autoresearch.openfeature": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/autoresearch/openfeature.py", + "tests/test_autoresearch/test_openfeature.py", + "docs/design/openfeature-autoresearch-experiments.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "no-bump: merge with main's product experiment layer (features.openfeature); design doc renamed to openfeature-autoresearch-experiments.md, provider import unguarded now that openfeature-sdk is a core dependency; resolution behavior unchanged" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; OpenFeature (CNCF) representation of typed autoresearch experiments: flagd flag-definition export (knob flags, experiment-id variants, JsonLogic targeting) plus fail-closed ExperimentFlagProvider over ExperimentSpec knobs" + } + ] + }, + "harness.autoresearch.experiment_campaign": { + "version": "v4", + "kind": "harness", + "paths": [ + "src/slm_training/autoresearch/experiment_campaign.py", + "src/slm_training/autoresearch/schemas.py", + "src/slm_training/autoresearch/storage.py", + "src/slm_training/autoresearch/engine.py", + "src/slm_training/autoresearch/rl_gate.py", + "src/slm_training/autoresearch/__init__.py", + "tests/test_autoresearch/test_experiment_campaign.py", + "tests/test_autoresearch/test_harness.py" + ], + "history": [ + { + "version": "v4", + "date": "2026-07-24", + "note": "Require AgentEvals-authoritative ship criteria rather than a self-reported AgentV boolean for RL readiness" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "Verify authoritative artifacts, outcome-boundary event binding, portable RL evidence, and atomic campaign locks" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "Reject boolean seed identifiers in governed result evidence before promotion completeness checks" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "SLM-337 immutable preregistered campaigns, pre-start digest binding, exploratory deviations, complete promotion claims, and verified RL evidence" + } + ] + }, + "harness.core": { + "version": "v9", + "kind": "harness", + "paths": [ + "src/slm_training/harness_core/" + ], + "history": [ + { + "version": "v9", + "date": "2026-07-24", + "note": "add SLM-291 EvidenceBundleV1, pluggable content-addressed evidence store, and PreregisteredClaimBarV1" + }, + { + "version": "v8", + "date": "2026-07-23", + "note": "add SLM-193 DSL-agnostic bit-exact state, candidate, closure, and bridge cache primitives" + }, + { + "version": "v7", + "date": "2026-07-23", + "note": "Keep explicit historical result blocks visible while rejecting arbitrary nested metrics from malformed canonical records" + }, + { + "version": "v6", + "date": "2026-07-23", + "note": "Expose deterministic JSON pointers from the canonical historical experiment-record normalizer" + }, + { + "version": "v5", + "date": "2026-07-23", + "note": "Validate finite real metrics and non-negative integral evidence counts before gate comparisons" + }, + { + "version": "v4", + "date": "2026-07-23", + "note": "Return an exact evidence, integrity, quality, and runtime failure partition from the generic gate engine" + }, + { + "version": "v3", + "date": "2026-07-22", + "note": "reject canonical experiment records with missing root suites instead of promoting nested diagnostic metric blocks" + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "prefer explicit meaningful-program v1 metrics over the legacy parse-rate fallback when normalizing committed experiment records" + }, + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration; frozen DSL-agnostic core extracted structurally (versioning, lineage, checkpoint_reference, gate/promotion engines, scaling math, record_schema/eval_cache/score_policy) with behavior pinned by tests/test_harness_core" + } + ] + }, + "harness.flags": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/flags/", + "src/slm_training/harnesses/model_build/experiment_flags.py", + "docs/design/openfeature-research-levers.md", + "tests/test_flags/test_openfeature_experiments.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "SLM-342: add isolated OpenFeature-compatible research-lever evaluation, assignment metadata, and OFREP read surface" + } + ] + }, + "harness_core.evidence_bundle": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harness_core/evidence_bundle.py", + "src/slm_training/harnesses/model_build/evidence_bundle_census.py", + "scripts/verify_evidence_bundles.py", + "tests/test_harness_core/test_evidence_bundle.py", + "tests/test_scripts/test_verify_evidence_bundles.py", + "docs/design/iter-slm291-evidence-bundles-20260724.json", + "docs/design/iter-slm291-evidence-bundles-20260724.md", + "docs/design/slm291-evidence-bar-adjudications-20260724.json" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-291 content-addressed EvidenceBundleV1 + CAS store, fail-closed verifier and model-card census, PreregisteredClaimBarV1, append-only retroactive adjudications" + } + ] + }, + "harness.distill": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/distill/legal_set_kl.py", + "src/slm_training/harnesses/distill/legal_set_teacher_trace.py", + "src/slm_training/harnesses/distill/semantic_repair.py", + "scripts/run_spv2_05_semantic_repair_fixture.py", + "tests/test_harnesses/distill/test_legal_set_kl.py", + "tests/test_harnesses/distill/test_semantic_repair.py", + "docs/design/iter-spv2-05-semantic-repair-20260720.md", + "docs/design/iter-spv2-05-semantic-repair-20260720.json" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-20", + "note": "add SLM-153 SPV2-05 verifier-guided minimal semantic repair fixture wiring baseline" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-151 SPV2-03 dense legal-set distillation fixture wiring baseline" + } + ] + }, + "harness.experiments": { + "version": "v83", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/" + ], + "history": [ + { + "version": "v83", + "date": "2026-07-24", + "note": "Implement SLM-294 frontier chunked execution and scoring pass in the external-ceiling harness" + }, + { + "version": "v82", + "date": "2026-07-24", + "note": "add pinned offline-only Transformers transport for SLM-266 local teacher generation" + }, + { + "version": "v81", + "date": "2026-07-24", + "note": "no-bump: correct SLM-262 completed-job test isolation and export history metadata without changing harness behavior" + }, + { + "version": "v81", + "date": "2026-07-24", + "note": "no-bump: add SLM-262 lazy experiment exports without changing existing experiment behavior" + }, + { + "version": "v81", + "date": "2026-07-24", + "note": "require durable cross-family automatic judge evidence for SLM-266 deep admission" + }, + { + "version": "v80", + "date": "2026-07-24", + "note": "add preflighted provider executor and immutable campaign raw-attempt archive for SLM-266" + }, + { + "version": "v79", + "date": "2026-07-24", + "note": "materialize deep SLM-266 admission into canonical train snapshots with raw archive pointers" + }, + { + "version": "v78", + "date": "2026-07-24", + "note": "remove human-audit gate from SLM-266 hands-off principal admission" + }, + { + "version": "v77", + "date": "2026-07-24", + "note": "add fail-closed SLM-266 teacher-program request and admission harness" + }, + { + "version": "v76", + "date": "2026-07-24", + "note": "add deterministic SLM-207 valid-edit-flow closeout harness" + }, + { + "version": "v75", + "date": "2026-07-23", + "note": "add SLM-194 legality-neutral candidate proposal and exact-fallback amortization fixture" + }, + { + "version": "v74", + "date": "2026-07-23", + "note": "integrate SLM-193 FFE3-02 bit-exact flow-cache fixture harness on current main" + }, + { + "version": "v73", + "date": "2026-07-23", + "note": "integrate SLM-192 FFE3-01 flow-pipeline cost-profile fixture harness on current main" + }, + { + "version": "v72", + "date": "2026-07-23", + "note": "add SLM-230 bounded recurrence observability and evaluation-only exit contracts" + }, + { + "version": "v71", + "date": "2026-07-23", + "note": "publish SLM-228 SpectralDispositionV1 and guard lineage/promotion from rejected spectral mechanisms" + }, + { + "version": "v70", + "date": "2026-07-23", + "note": "add SLM-226 width and finite-size null boundary with fail-closed AbsoluteSpectralTargetGateV1" + }, + { + "version": "v69", + "date": "2026-07-23", + "note": "add SLM-220 activation-side causal restriction energy, exact and JVP estimators, null controls, and fail-closed fixture retrospective" + }, + { + "version": "v68", + "date": "2026-07-23", + "note": "add SLM-219 deterministic prefix checkpoint trajectory, canonical spectral diagnostics, pinned WeightWatcher comparison, and fail-closed warning retrospective" + }, + { + "version": "v67", + "date": "2026-07-23", + "note": "add SLM-218 cross-attention and parent-child subspace retrospective owners" + }, + { + "version": "v66", + "date": "2026-07-23", + "note": "add exact-state decision-conditioned functional spectral diagnostics for SLM-217" + }, + { + "version": "v65", + "date": "2026-07-23", + "note": "add the bounded SLM-216 fixed-token spectral regime matrix and fail-closed gate" + }, + { + "version": "v64", + "date": "2026-07-23", + "note": "publish the fail-closed SLM-213 semantic-floor claim authorization gate" + }, + { + "version": "v63", + "date": "2026-07-23", + "note": "repair canonical InsertStatement application for replay-exact SLM-196 bridge corpus construction" + }, + { + "version": "v62", + "date": "2026-07-23", + "note": "Derive promotion decisions from independently verified campaign results and plumb governance through canonical entrypoints" + }, + { + "version": "v61", + "date": "2026-07-23", + "note": "Require campaign-governed promotion, integrity-check legacy frozen manifests, and bridge the power-protocol fixture to ExperimentCampaignV1" + }, + { + "version": "v60", + "date": "2026-07-21", + "note": "no-bump: source the already-v60 two-minute fixture budget from the canonical lever registry; emitted behavior is unchanged" + }, + { + "version": "v60", + "date": "2026-07-21", + "note": "hard-cap all experiment fixture wall-budget metadata at two minutes" + }, + { + "version": "v59", + "date": "2026-07-21", + "note": "add SLM-191 FFE2-03 termination-policy protocol and fixture matrix lazy re-exports" + }, + { + "version": "v58", + "date": "2026-07-21", + "note": "add SLM-190 FFE2-02 exact CTMC reference fixture harness lazy re-exports" + }, + { + "version": "v57", + "date": "2026-07-21", + "note": "add SLM-189 FFE2-01 bridge-planner fixture harness lazy re-exports" + }, + { + "version": "v56", + "date": "2026-07-21", + "note": "add SLM-212 SDE5-05 constraint-debt routing fixture harness lazy re-exports" + }, + { + "version": "v55", + "date": "2026-07-21", + "note": "no-bump: re-export SLM-222 MuonBaselineArm/MuonBaselineReport/run_muon_baseline_fixture from experiments package; existing experiment harnesses unchanged" + }, + { + "version": "v55", + "date": "2026-07-21", + "note": "add SLM-210 SDE5-03 floor-escape matrix wiring/fixture harness extending SLM-209 debt-targeted curriculum owner" + }, + { + "version": "v54", + "date": "2026-07-21", + "note": "add SLM-215 NCS0-02 SpectralAtlasV1 wiring/fixture harness for null-calibrated checkpoint atlas and outcome joins" + }, + { + "version": "v53", + "date": "2026-07-21", + "note": "add SLM-214 NCS0-01 SpectralSnapshotV1 wiring/fixture harness with per-shape randomized-ESD null calibration" + }, + { + "version": "v52", + "date": "2026-07-21", + "note": "add SLM-188 FFE1-02 edit-algebra reachability, canonical invariance, and transition-certificate wiring/fixture harness" + }, + { + "version": "v51", + "date": "2026-07-21", + "note": "add SLM-187 FFE1-01 topology solver/runtime transition parity wiring/fixture harness" + }, + { + "version": "v50", + "date": "2026-07-21", + "note": "no-bump: add SLM-186 FFE0-04 verified-utility sub-harness; no change to existing experiment harness behavior" + }, + { + "version": "v50", + "date": "2026-07-20", + "note": "add SLM-209 SDE5-02 debt-targeted semantic exposure curriculum wiring/fixture harness" + }, + { + "version": "v49", + "date": "2026-07-20", + "note": "no-bump: SLM-185 reuses harness.experiments constants (MATRIX_SET/MATRIX_VERSION/EXPERIMENT_ID pattern) only; no harness behavior changed" + }, + { + "version": "v49", + "date": "2026-07-20", + "note": "add SLM-184 single-touch confirmation firewall and preregistered claim manifest wiring/fixture harness" + }, + { + "version": "v48", + "date": "2026-07-20", + "note": "add SLM-183 PQR powered cluster-aware confirmation protocol wiring/fixture harness" + }, + { + "version": "v47", + "date": "2026-07-20", + "note": "add SLM-176 P14 action-shortlist retrieve-then-rerank wiring/fixture harness" + }, + { + "version": "v46", + "date": "2026-07-20", + "note": "add SLM-174 SDE2-07 action-alias generalization wiring/fixture harness" + }, + { + "version": "v45", + "date": "2026-07-20", + "note": "add SLM-172 SDE2-05 render-equivalence wiring/fixture harness" + }, + { + "version": "v44", + "date": "2026-07-20", + "note": "add SLM-170 SDE2-03 exposure-targeted rare-action sampling wiring/fixture harness" + }, + { + "version": "v43", + "date": "2026-07-20", + "note": "add SLM-168 public structured contract-index pointer wiring/fixture harness" + }, + { + "version": "v42", + "date": "2026-07-20", + "note": "add SLM-167 zero-training sparse-action ceiling wiring/fixture harness" + }, + { + "version": "v41", + "date": "2026-07-20", + "note": "add SLM-166 semantic connector capacity wiring/fixture harness" + }, + { + "version": "v40", + "date": "2026-07-20", + "note": "add SLM-165 2x2x2 interaction factorial wiring/fixture harness" + }, + { + "version": "v39", + "date": "2026-07-20", + "note": "add SLM-164 targeted legal-sibling contrast margin wiring/fixture harness" + }, + { + "version": "v38", + "date": "2026-07-20", + "note": "add SLM-163 schema-description action-embedding wiring/fixture harness" + }, + { + "version": "v37", + "date": "2026-07-20", + "note": "update SLM-160 SPV4-02 disposition rationale for SLM-157 now that the fixture evidence document exists" + }, + { + "version": "v36", + "date": "2026-07-20", + "note": "add SLM-157 SPV3-04 flow / consistency / trajectory-imitation wiring/fixture harness" + }, + { + "version": "v35", + "date": "2026-07-20", + "note": "add SLM-160 SPV4-02 causal architecture disposition docs/spec harness" + }, + { + "version": "v34", + "date": "2026-07-20", + "note": "add SLM-159 SPV4-01 cross-DSL semantic-plan replication wiring/fixture harness" + }, + { + "version": "v33", + "date": "2026-07-20", + "note": "add SLM-158 SPV3-05 sequence-mixer comparison wiring/fixture harness" + }, + { + "version": "v32", + "date": "2026-07-20", + "note": "add SLM-156 SPV3-03 shared recursive SemanticPlanV1 refinement wiring/fixture harness" + }, + { + "version": "v31", + "date": "2026-07-20", + "note": "add SLM-148 SPV1-05 plan-conditioned X22 \u00d7 conflict-slice staged campaign wiring/fixture harness" + }, + { + "version": "v30", + "date": "2026-07-20", + "note": "add SLM-147 SPV1-04 X22 leakage-safe retrieved-valid-AST prototype seeding harness" + }, + { + "version": "v29", + "date": "2026-07-20", + "note": "add SLM-140 EFS4-04 causal synthesis manifest, loader, script, and tests" + }, + { + "version": "v28", + "date": "2026-07-20", + "note": "add SLM-146 SPV1-03 SemanticPlanV1 plan-compiler bridge fixture wiring harness" + }, + { + "version": "v27", + "date": "2026-07-20", + "note": "add SLM-135 EFS4-01 trailed-assumptions ablation fixture wiring harness" + }, + { + "version": "v26", + "date": "2026-07-20", + "note": "add SLM-180 SDE4-02 minimum-controller-capacity fixture wiring harness" + }, + { + "version": "v25", + "date": "2026-07-20", + "note": "no-bump: replace randomized string hash with stable action-index mapping in cap2-04 fixture; matrix semantics unchanged" + }, + { + "version": "v25", + "date": "2026-07-20", + "note": "add SLM-144 SPV1-01 archetype + role-set predictor fixture wiring harness" + }, + { + "version": "v24", + "date": "2026-07-19", + "note": "add SLM-133 EFS3-06 AST-sketch dedup \u00d7 choice-native retrieval factorial wiring/fixture harness" + }, + { + "version": "v23", + "date": "2026-07-19", + "note": "add SLM-130 EFS3-05 canonical AST deduplication and valid semantic-mode coverage wiring/fixture harness" + }, + { + "version": "v22", + "date": "2026-07-19", + "note": "add SLM-127 EFS3-04 contract-grounded candidate selector wiring/fixture harness" + }, + { + "version": "v21", + "date": "2026-07-19", + "note": "lazy-load B3 capacity exports so importing the experiment package and Vercel web entrypoint does not require optional torch" + }, + { + "version": "v20", + "date": "2026-07-19", + "note": "add SLM-124 EFS3-03 B3 surface-vs-choice capacity-ladder v2 manifest, fixture runner, and CLI wiring" + }, + { + "version": "v19", + "date": "2026-07-19", + "note": "add SLM-121 LDI1-02 causal PEFT FTPO manifest, fixture runner, and CLI wiring" + }, + { + "version": "v18", + "date": "2026-07-19", + "note": "add SLM-120 near-solved semantic corruption-curriculum manifest, fixture runner, and CLI wiring" + }, + { + "version": "v17", + "date": "2026-07-19", + "note": "add SLM-109 E228 exposure-ladder manifest, recipe-freeze hash, and plan/fixture CLI" + }, + { + "version": "v16", + "date": "2026-07-19", + "note": "add SLM-108 external-ceiling matrix manifest, fixture runner, and report rendering" + }, + { + "version": "v15", + "date": "2026-07-19", + "note": "no-bump: relocate efficiency_gain/scaling_fit to harness_core (old modules are sys.modules alias shims) and delegate promotion checks to harness_core.promotion_engine; HARD_CATEGORIES, the OpenUI ship-gate binding, and all public signatures unchanged" + }, + { + "version": "v15", + "date": "2026-07-19", + "note": "lazy-load torch-backed ladder re-exports so the read-only Vercel web entrypoint remains torch-free" + }, + { + "version": "v14", + "date": "2026-07-19", + "note": "fix(cap3-05): make equal-byte planning honest and reconcile cost ledger" + }, + { + "version": "v13", + "date": "2026-07-19", + "note": "add CAP3-05 (SLM-94) equal-byte width \u00d7 precision ladder planner and dry-run manifest support" + }, + { + "version": "v12", + "date": "2026-07-19", + "note": "add SDE3-03 (SLM-177) proxy-metric calibration activation/budget/feature manifest dataclass, builder/validator, and CLI" + }, + { + "version": "v11", + "date": "2026-07-19", + "note": "add SDE3-04 constraint-backend benchmark manifest dataclass, builder/validator, and CLI for frozen grammar-backend comparison plan" + }, + { + "version": "v10", + "date": "2026-07-19", + "note": "add SDE4-01 (SLM-179) scaffold-distillation activation manifest dataclass, builder/validator, and CLI" + }, + { + "version": "v9", + "date": "2026-07-19", + "note": "add SDE4-04 pretrained-denoiser activation/candidate/budget manifest (wiring slice)" + }, + { + "version": "v8", + "date": "2026-07-19", + "note": "add SDE4-03 teacher-paraphrase activation/budget manifest and canonical request renderer (wiring slice)" + }, + { + "version": "v7", + "date": "2026-07-19", + "note": "add EFS2-03 conflict-slice repair harness (ConflictSliceV1, repair policies, deterministic fixtures, matched-budget comparison) on top of EFS0-05/CAP5 manifests" + }, + { + "version": "v6", + "date": "2026-07-19", + "note": "add EFS0-05 rejected-lever registry (RejectedLeverV1, ReAdjudicationRowV1, paired five-seed classification, autoresearch evidence items) on top of CAP5-02/CAP5-01 manifests" + }, + { + "version": "v5", + "date": "2026-07-19", + "note": "add CAP5-02 campaign manifest dataclass, builder/validator, and CLI for preregistered quality/cost arms" + }, + { + "version": "v4", + "date": "2026-07-19", + "note": "add GrammarProfile dataclass, manifest builder/validator, and CLI for CAP5-01 grammar/profile family summaries" + }, + { + "version": "v3", + "date": "2026-07-18", + "note": "lazy-load Torch-only equal-byte planning dependencies so the web runtime remains import-safe" + }, + { + "version": "v2", + "date": "2026-07-18", + "note": "correct equal-byte arm token budgets and fail closed on reference-only low-bit training" + }, + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration (ladder, promotion, scaling fits; verified-solver files are claimed by matrix.verified_solver)" + } + ] + }, + "harness.experiments.external_ceiling": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/models/external_scorer.py", + "scripts/run_external_ceiling.py", + "src/slm_training/harnesses/experiments/external_ceiling_matrix.py", + "tests/test_scripts/test_run_external_ceiling_frontier.py" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-24", + "note": "add arm-A tiny-baseline chunk runner, locked-threshold disposition mode with EvidenceBundleV1 plus AgentV, per-arm dtype and max_new_tokens controls" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-294 external-ceiling frontier execution (chunked resume, atomic raw JSONL, Wilson scoreboard) and scorer generate()" + } + ] + }, + "harness.experiments.ast_sketch_retrieval": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/ast_sketch_retrieval_factorial.py", + "scripts/run_ast_sketch_retrieval_factorial.py", + "tests/test_harnesses/experiments/test_ast_sketch_retrieval_factorial.py", + "tests/test_scripts/test_run_ast_sketch_retrieval_factorial.py" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-21", + "note": "no-bump: source the already-v2 wall budget from the canonical lever registry; emitted behavior is unchanged" + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "hard-cap AST-sketch retrieval fixture wall-budget metadata at two minutes" + }, + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration; SLM-133 EFS3-06 AST-sketch dedup \u00d7 choice-native retrieval factorial wiring/fixture harness" + } + ] + }, + "harness.experiments.candidate_selector": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/candidate_selector.py", + "scripts/run_candidate_selector.py", + "tests/test_harnesses/experiments/test_candidate_selector.py", + "tests/test_scripts/test_run_candidate_selector.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration; SLM-127 EFS3-04 contract-grounded candidate selector wiring/fixture harness" + } + ] + }, + "harness.experiments.canonical_ast_dedup": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/canonical_ast_dedup.py", + "src/slm_training/dsl/grammar/fastpath/lattice_search.py", + "scripts/run_canonical_ast_dedup.py", + "tests/test_harnesses/experiments/test_canonical_ast_dedup.py", + "tests/test_scripts/test_run_canonical_ast_dedup.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration; SLM-130 EFS3-05 canonical AST deduplication and valid semantic-mode coverage wiring/fixture harness" + } + ] + }, + "harness.experiments.claim_manifest": { + "version": "v3", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/claim_manifest.py", + "scripts/audit_experiment_firewall.py", + "tests/test_harnesses/experiments/test_claim_manifest.py", + "tests/test_scripts/test_audit_experiment_firewall.py", + "docs/design/iter-slm184-claim-manifest-20260720.json", + "docs/design/iter-slm184-claim-manifest-20260720.md" + ], + "history": [ + { + "version": "v3", + "date": "2026-07-23", + "note": "Permit idempotent replay of an identical intact frozen manifest while rejecting mutation and tampering" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "Make frozen claim manifests create-once and verify their canonical content digest before confirmation access" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-184 single-touch confirmation firewall and preregistered claim manifest wiring/fixture harness" + } + ] + }, + "harness.experiments.semantic_floor_gate": { + "version": "v1", + "kind": "gate", + "paths": [ + "src/slm_training/harnesses/experiments/semantic_floor_gate.py", + "scripts/publish_semantic_floor_gate.py", + "tests/test_harnesses/experiments/test_semantic_floor_gate.py", + "tests/test_scripts/test_publish_semantic_floor_gate.py", + "docs/design/semantic-floor-gate-v1.json", + "docs/design/semantic-floor-gate-v1.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "normalize the component version and refresh registered identities while preserving the evidence timestamp" + }, + { + "version": "sde5-06-v1", + "date": "2026-07-23", + "note": "initial SLM-213 SemanticFloorGateV1 evidence aggregation and claim authorization" + } + ] + }, + "harness.experiments.efs4_04_causal_synthesis": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/efs4_04_causal_synthesis.py", + "scripts/synthesize_efs_campaign.py", + "tests/test_harnesses/experiments/test_efs4_04_causal_synthesis.py", + "tests/test_scripts/test_synthesize_efs_campaign.py", + "docs/design/evidence-first-semantic-slm-campaign-v1.json", + "docs/design/iter-efs4-04-causal-synthesis-20260720.json", + "docs/design/iter-efs4-04-causal-synthesis-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: make --write-default-manifest exit after writing the manifest so tests do not overwrite committed design artifacts" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: add regression tests for the campaign manifest, synthesis loader, and CLI; harness behavior unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration of SLM-140 EFS4-04 Evidence-First Semantic SLM causal synthesis manifest and loader" + } + ] + }, + "harness.experiments.sde4_02_min_controller_capacity": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/sde4_02_min_controller_capacity.py", + "scripts/run_sde4_02_min_controller_capacity.py", + "tests/test_harnesses/experiments/test_sde4_02_min_controller_capacity.py", + "tests/test_scripts/test_run_sde4_02_min_controller_capacity.py", + "docs/design/iter-sde4-02-min-controller-capacity-20260720.json", + "docs/design/iter-sde4-02-min-controller-capacity-20260720.md" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-21", + "note": "no-bump: source the already-v2 wall budget from the canonical lever registry; emitted behavior is unchanged" + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "hard-cap minimum-controller fixture wall-budget metadata at two minutes" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact timestamp after sibling fixture CLI test run; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design JSON version stamp after harness.experiments v27 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact timestamps after fixture CLI rerun; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration of SLM-180 SDE4-02 minimum-controller-capacity fixture wiring harness" + } + ] + }, + "harness.experiments.sde5_floor_escape_matrix": { + "version": "sde5-03-v1", + "kind": "harness", + "paths": [ + "tests/test_harnesses/experiments/test_sde5_floor_escape_matrix.py", + "docs/design/sde5-floor-escape-matrix-results.json", + "docs/design/sde5-floor-escape-matrix-results.md" + ], + "history": [ + { + "version": "sde5-03-v1", + "date": "2026-07-21", + "note": "initial registration; SLM-210 SDE5-03 prompt-plan \u00d7 grammar-mass \u00d7 high-debt exposure floor-escape matrix wiring/fixture harness" + } + ] + }, + "harness.experiments.semantic_regret": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/semantic_regret_matrix.py", + "scripts/run_semantic_regret_fixture.py", + "tests/test_harnesses/experiments/test_semantic_regret_matrix.py", + "tests/test_scripts/test_run_semantic_regret_fixture.py", + "docs/design/iter-spv0-03-semantic-regret-20260719.json", + "docs/design/iter-spv0-03-semantic-regret-20260719.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: append run-record rows to design markdown and normalize trailing whitespace in fixture runner markdown string; harness behavior, thresholds, and report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact timestamp after sibling fixture CLI test run; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design JSON version stamp after harness.experiments v27 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design JSON version stamp after harness.experiments v26 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design JSON version stamp after harness.experiments v25 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration of semantic regret decomposition fixture harness (SLM-143)" + } + ] + }, + "harness.experiments.slm135_trailed_assumptions_ablation": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm135_trailed_assumptions_ablation.py", + "scripts/run_slm135_trailed_assumptions_fixture.py", + "tests/test_harnesses/experiments/test_slm135_trailed_assumptions_ablation.py", + "tests/test_scripts/test_run_slm135_trailed_assumptions_fixture.py", + "docs/design/iter-slm135-trailed-assumptions-20260720.json", + "docs/design/iter-slm135-trailed-assumptions-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact timestamp after sibling fixture CLI test run; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact timestamp after fixture CLI rerun; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration of SLM-135 EFS4-01 trailed-assumptions ablation fixture wiring harness" + } + ] + }, + "harness.experiments.slm144_plan_predictor": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm144_plan_predictor_matrix.py", + "src/slm_training/models/semantic_plan_predictor.py", + "src/slm_training/data/semantic_plan/corpus.py", + "scripts/run_slm144_plan_predictor_fixture.py", + "tests/test_harnesses/experiments/test_slm144_plan_predictor_matrix.py", + "tests/test_models/test_semantic_plan_predictor.py", + "tests/test_scripts/test_run_slm144_plan_predictor_fixture.py", + "docs/design/iter-slm144-plan-predictor-20260720.json", + "docs/design/iter-slm144-plan-predictor-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: extend build_fixture_plan_corpus with optional root/leaf component pools; default behavior, extracted plans, and SLM-144 fixture metrics unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact timestamp after sibling fixture CLI test run; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design JSON version stamp after harness.experiments v27 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design JSON version stamp after harness.experiments v26 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact timestamps after fixture CLI rerun; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration of SLM-144 SPV1-01 archetype + role-set predictor fixture harness" + } + ] + }, + "harness.experiments.slm146_plan_compiler": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/data/semantic_plan/compiler.py", + "src/slm_training/harnesses/experiments/slm146_semantic_plan_compiler.py", + "scripts/run_slm146_semantic_plan_compiler_fixture.py", + "tests/test_data/test_semantic_plan_extraction/test_compiler.py", + "tests/test_harnesses/experiments/test_slm146_semantic_plan_compiler.py", + "tests/test_scripts/test_run_slm146_semantic_plan_compiler_fixture.py", + "docs/design/iter-slm146-semantic-plan-compiler-20260720.json", + "docs/design/iter-slm146-semantic-plan-compiler-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refine fixture CLI so plan-only mode no longer overwrites committed design artifacts; refresh fixture design JSON/Markdown" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration of SLM-146 SPV1-03 SemanticPlanV1 to compiler-choice and X22 tree-edit bridge" + } + ] + }, + "harness.experiments.slm147_x22_retrieval": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm147_x22_retrieval.py", + "scripts/run_slm147_x22_retrieval.py", + "tests/test_harnesses/experiments/test_slm147_x22_retrieval.py", + "tests/test_scripts/test_run_slm147_x22_retrieval.py", + "docs/design/iter-slm147-x22-retrieval-20260720.json", + "docs/design/iter-slm147-x22-retrieval-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamp after harness.experiments v30 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration of SLM-147 SPV1-04 X22 leakage-safe retrieved-valid-AST prototype seeding harness" + } + ] + }, + "harness.experiments.slm148_x22_conflict_campaign": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm148_x22_conflict_campaign.py", + "scripts/run_slm148_x22_conflict_campaign.py", + "tests/test_harnesses/experiments/test_slm148_x22_conflict_campaign.py", + "tests/test_scripts/test_run_slm148_x22_conflict_campaign.py", + "docs/design/iter-slm148-x22-conflict-campaign-20260720.json", + "docs/design/iter-slm148-x22-conflict-campaign-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration of SLM-148 SPV1-05 plan-conditioned X22 \u00d7 conflict-slice staged campaign wiring/fixture harness" + } + ] + }, + "harness.experiments.slm155_factorization_comparison": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm155_factorization_comparison.py", + "scripts/run_slm155_factorization_comparison_fixture.py", + "scripts/run_slm297_unified_rescore.py", + "tests/test_harnesses/experiments/test_slm155_factorization_comparison.py", + "tests/test_scripts/test_run_slm155_factorization_comparison_fixture.py", + "tests/test_scripts/test_run_slm297_unified_rescore.py", + "docs/design/iter-slm155-factorization-comparison-20260720.json", + "docs/design/iter-slm155-factorization-comparison-20260720.md", + "docs/design/iter-slm297-unified-rescore-20260724.json", + "docs/design/iter-slm297-unified-rescore-20260724.md" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-24", + "note": "SLM-297: retain per-example final programs in FactorizationRecord (AR via _program_from_actions, X22 seed + program_note, hybrid source), add FinalProgramOutcomeV1 unified final-program scoring + transition/paired-table helpers, run_fixture_campaign_with_records, and the run_slm297_unified_rescore CLI; aggregate rows unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-155 SPV3-02 matched AR vs plan-conditioned X22 factorization comparison wiring/fixture harness" + } + ] + }, + "harness.experiments.slm156_plan_refinement": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm156_plan_refinement.py", + "scripts/run_slm156_plan_refinement_fixture.py", + "tests/test_harnesses/experiments/test_slm156_plan_refinement.py", + "tests/test_scripts/test_run_slm156_plan_refinement_fixture.py", + "docs/design/iter-slm156-plan-refinement-20260720.json", + "docs/design/iter-slm156-plan-refinement-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-156 SPV3-03 shared recursive SemanticPlanV1 refinement wiring/fixture harness" + } + ] + }, + "harness.experiments.slm157_flow_consistency": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm157_flow_consistency.py", + "scripts/run_slm157_flow_consistency_fixture.py", + "tests/test_harnesses/experiments/test_slm157_flow_consistency.py", + "tests/test_scripts/test_run_slm157_flow_consistency_fixture.py", + "docs/design/iter-slm157-flow-consistency-20260720.json", + "docs/design/iter-slm157-flow-consistency-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-157 SPV3-04 flow / consistency / trajectory-imitation wiring/fixture harness" + } + ] + }, + "harness.experiments.slm158_mixer_comparison": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm158_mixer_comparison.py", + "scripts/run_slm158_mixer_comparison_fixture.py", + "tests/test_harnesses/experiments/test_slm158_mixer_comparison.py", + "tests/test_scripts/test_run_slm158_mixer_comparison_fixture.py", + "docs/design/iter-slm158-mixer-comparison-20260720.json", + "docs/design/iter-slm158-mixer-comparison-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-158 SPV3-05 sequence-mixer comparison wiring/fixture harness" + } + ] + }, + "harness.experiments.slm159_cross_dsl_replication": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm159_cross_dsl_replication.py", + "scripts/run_slm159_cross_dsl_replication_fixture.py", + "tests/test_harnesses/experiments/test_slm159_cross_dsl_replication.py", + "tests/test_scripts/test_run_slm159_cross_dsl_replication_fixture.py", + "docs/design/iter-slm159-cross-dsl-replication-20260720.json", + "docs/design/iter-slm159-cross-dsl-replication-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-159 SPV4-01 cross-DSL semantic-plan replication wiring/fixture harness" + } + ] + }, + "harness.experiments.slm160_spv_disposition": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm160_spv_disposition.py", + "scripts/run_slm160_spv_disposition.py", + "tests/test_harnesses/experiments/test_slm160_spv_disposition.py", + "tests/test_scripts/test_run_slm160_spv_disposition.py", + "docs/design/iter-slm160-spv-disposition-20260720.json", + "docs/design/iter-slm160-spv-disposition-20260720.md" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v2", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v2", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v2", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v2", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v2", + "date": "2026-07-20", + "note": "update SLM-157 flow/consistency disposition rationale now that the fixture evidence document exists" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-160 SPV4-02 causal architecture disposition docs/spec harness" + } + ] + }, + "harness.experiments.slm163_schema_action_embedding": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm163_schema_action_embedding.py", + "scripts/run_slm163_schema_action_embedding_fixture.py", + "tests/test_harnesses/experiments/test_slm163_schema_action_embedding.py", + "tests/test_scripts/test_run_slm163_schema_action_embedding_fixture.py", + "docs/design/iter-slm163-schema-action-embedding-20260720.json", + "docs/design/iter-slm163-schema-action-embedding-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-163 schema-description action-embedding wiring/fixture harness" + } + ] + }, + "harness.experiments.slm164_targeted_margin": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm164_targeted_margin.py", + "scripts/run_slm164_targeted_margin_fixture.py", + "tests/test_harnesses/experiments/test_slm164_targeted_margin.py", + "tests/test_scripts/test_run_slm164_targeted_margin_fixture.py", + "docs/design/iter-slm164-targeted-margin-20260720.json", + "docs/design/iter-slm164-targeted-margin-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-164 confusion-targeted legal-sibling contrast margin wiring/fixture harness" + } + ] + }, + "harness.experiments.slm165_interaction_factorial": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm165_interaction_factorial.py", + "scripts/run_slm165_interaction_factorial_fixture.py", + "tests/test_harnesses/experiments/test_slm165_interaction_factorial.py", + "tests/test_scripts/test_run_slm165_interaction_factorial_fixture.py", + "docs/design/iter-slm165-interaction-factorial-20260720.json", + "docs/design/iter-slm165-interaction-factorial-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after commit; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-165 2x2x2 interaction factorial wiring/fixture harness" + } + ] + }, + "harness.experiments.slm166_connector_capacity": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm166_connector_capacity.py", + "scripts/run_slm166_connector_capacity_fixture.py", + "tests/test_harnesses/experiments/test_slm166_connector_capacity.py", + "tests/test_scripts/test_run_slm166_connector_capacity_fixture.py", + "docs/design/iter-slm166-connector-capacity-20260720.json", + "docs/design/iter-slm166-connector-capacity-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after commit; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-166 semantic connector capacity wiring/fixture harness" + } + ] + }, + "harness.experiments.slm167_zero_training_sparse_ceiling": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm167_zero_training_sparse_ceiling.py", + "scripts/run_slm167_zero_training_sparse_ceiling_fixture.py", + "tests/test_harnesses/experiments/test_slm167_zero_training_sparse_ceiling.py", + "tests/test_scripts/test_run_slm167_zero_training_sparse_ceiling_fixture.py", + "docs/design/iter-slm167-zero-training-sparse-ceiling-20260720.json", + "docs/design/iter-slm167-zero-training-sparse-ceiling-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after commit amend; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-167 zero-training sparse-action ceiling wiring/fixture harness" + } + ] + }, + "harness.experiments.slm168_public_structured_contract_pointer": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm168_public_structured_contract_pointer.py", + "scripts/run_slm168_public_structured_contract_pointer_fixture.py", + "tests/test_harnesses/experiments/test_slm168_public_structured_contract_pointer.py", + "tests/test_scripts/test_run_slm168_public_structured_contract_pointer_fixture.py", + "docs/design/iter-slm168-public-structured-contract-pointer-20260720.json", + "docs/design/iter-slm168-public-structured-contract-pointer-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-168 public structured contract-index pointer wiring/fixture harness" + } + ] + }, + "harness.experiments.slm170_exposure_targeted_rare_action": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm170_exposure_targeted_rare_action.py", + "scripts/run_slm170_exposure_targeted_rare_action_fixture.py", + "tests/test_harnesses/experiments/test_slm170_exposure_targeted_rare_action.py", + "tests/test_scripts/test_run_slm170_exposure_targeted_rare_action_fixture.py", + "docs/design/iter-slm170-exposure-targeted-rare-action-20260720.json", + "docs/design/iter-slm170-exposure-targeted-rare-action-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-170 SDE2-03 exposure-targeted rare-action sampling wiring/fixture harness" + } + ] + }, + "harness.experiments.slm172_render_equivalence": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm172_render_equivalence.py", + "scripts/run_slm172_render_equivalence_fixture.py", + "tests/test_harnesses/experiments/test_slm172_render_equivalence.py", + "tests/test_scripts/test_run_slm172_render_equivalence_fixture.py", + "docs/design/iter-slm172-render-equivalence-20260720.json", + "docs/design/iter-slm172-render-equivalence-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-172 SDE2-05 render-equivalence wiring/fixture harness" + } + ] + }, + "harness.experiments.slm174_action_alias_generalization": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm174_action_alias_generalization.py", + "scripts/run_slm174_action_alias_generalization_fixture.py", + "tests/test_harnesses/experiments/test_slm174_action_alias_generalization.py", + "tests/test_scripts/test_run_slm174_action_alias_generalization_fixture.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-174 SDE2-07 action-alias generalization fixture harness" + } + ] + }, + "harness.experiments.slm176_action_shortlist_rerank": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm176_action_shortlist_rerank.py", + "scripts/run_slm176_action_shortlist_rerank_fixture.py", + "tests/test_harnesses/experiments/test_slm176_action_shortlist_rerank.py", + "tests/test_scripts/test_run_slm176_action_shortlist_rerank_fixture.py", + "docs/design/iter-slm176-action-shortlist-rerank-20260720.json", + "docs/design/iter-slm176-action-shortlist-rerank-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-176 P14 action-shortlist retrieve-then-rerank fixture harness" + } + ] + }, + "harness.experiments.reserved_operator_baseline": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/reserved_operator_baseline.py", + "scripts/run_reserved_operator_baseline.py", + "tests/test_harnesses/experiments/test_reserved_operator_baseline.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial E803 capacity-matched result-only, operator-only, and operator-plus-result hashed-token experiment with causal choice and compiler-membership evidence" + } + ] + }, + "harness.experiments.slm183_power_protocol": { + "version": "v3", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm183_power_protocol.py", + "scripts/run_flow_power_protocol.py", + "tests/test_harnesses/experiments/test_slm183_power_protocol.py", + "tests/test_scripts/test_run_flow_power_protocol_fixture.py", + "docs/design/iter-slm183-power-protocol-20260720.json", + "docs/design/iter-slm183-power-protocol-20260720.md" + ], + "history": [ + { + "version": "v3", + "date": "2026-07-23", + "note": "Run exact paired control and candidate arms under a pre-start campaign lock with opt-in canonical doc writes" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "Embed the canonical ExperimentCampaignV1 preregistration with binder-reference fallback and Holm family metadata" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: add seed_contrast field to PowerProtocolReport and ruff style fixes; behavior-neutral" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-183 PQR powered cluster-aware confirmation protocol fixture harness" + } + ] + }, + "harness.experiments.slm186_verified_utility": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm186_verified_utility.py", + "scripts/run_verified_utility_audit.py", + "tests/test_harnesses/experiments/test_slm186_verified_utility.py", + "tests/test_scripts/test_run_verified_utility_audit.py", + "docs/design/iter-slm186-verified-utility-20260721.json", + "docs/design/iter-slm186-verified-utility-20260721.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-186 FFE0-04 verified-utility ladder and Goodhart canary audit wiring/fixture harness" + } + ] + }, + "harness.experiments.slm187_topology_parity": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm187_topology_parity.py", + "scripts/run_slm187_topology_parity_fixture.py", + "tests/test_harnesses/experiments/test_slm187_topology_parity.py", + "tests/test_scripts/test_run_slm187_topology_parity_fixture.py", + "docs/design/iter-slm187-topology-parity-20260721.json", + "docs/design/iter-slm187-topology-parity-20260721.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-187 FFE1-01 topology solver/runtime transition parity oracle, TopologyStateV2 carrier, and fixture harness" + } + ] + }, + "harness.experiments.slm188_edit_algebra": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm188_edit_algebra.py", + "scripts/audit_edit_reachability.py", + "tests/test_harnesses/experiments/test_slm188_edit_algebra.py", + "tests/test_scripts/test_audit_edit_reachability.py", + "docs/design/iter-slm188-edit-algebra-20260721.json", + "docs/design/iter-slm188-edit-algebra-20260721.md" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "apply missing-statement insertions with typed leaf or container payloads instead of rejecting them before dispatch" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-188 FFE1-02 edit-algebra reachability, canonical invariance, TransitionCertificateV1, and audit CLI" + } + ] + }, + "harness.experiments.slm189_bridge_planner": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm189_bridge_planner.py", + "tests/test_harnesses/experiments/test_slm189_bridge_planner.py", + "tests/test_scripts/test_run_bridge_planner_audit.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-189 FFE2-01 bridge planner fixture harness" + } + ] + }, + "harness.experiments.slm190_exact_flow": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm190_exact_flow.py", + "tests/test_harnesses/experiments/test_slm190_exact_flow.py", + "tests/test_scripts/test_run_exact_flow_fixture.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-190 FFE2-02 exact finite-state CTMC reference fixture harness" + } + ] + }, + "harness.experiments.slm191_termination_matrix": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm191_termination_matrix.py", + "tests/test_harnesses/experiments/test_slm191_termination_matrix.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-191 FFE2-03 termination-policy fixture matrix, TerminationManifestV1, and calibration instrumentation" + } + ] + }, + "harness.experiments.slm262_gpu_reference_run": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm262_gpu_reference.py", + "scripts/run_gpu_reference.py", + "tests/test_harnesses/experiments/test_slm262_gpu_reference.py", + "tests/test_scripts/test_run_gpu_reference.py", + "docs/design/gpu-reference-run.md", + "docs/design/iter-slm262-gpu-reference-run-20260721.json" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: isolate completed-job reconciliation unit test from external bucket tooling" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-262 durable GPU train checkpoint eval reference-run manifest and adapter" + } + ] + }, + "harness.experiments.slm197_direct_bridge_policy": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm197_direct_bridge_policy.py", + "src/slm_training/models/legal_edit_scorer.py", + "scripts/run_direct_bridge_policy_matrix.py", + "tests/test_models/test_legal_edit_scorer.py", + "tests/test_harnesses/experiments/test_slm197_direct_bridge_policy.py", + "tests/test_scripts/test_run_direct_bridge_policy_matrix.py", + "docs/design/iter-slm197-direct-bridge-policy-20260723.json", + "docs/design/iter-slm197-direct-bridge-policy-20260723.md", + "docs/design/iter-slm197-direct-bridge-policy-agentv-20260723/", + "docs/design/quality-experiment-matrix.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: append corrected historical E615 fixture evidence to the shared quality matrix without changing the SLM-197 scorer or experiment" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: append the SLM-236 blocked RSC4 disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: append the SLM-233 gate-bounded matched recursive campaign to the shared quality matrix without changing the SLM-197 scorer or experiment" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: append the SLM-243 measured recursive-update architecture disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: append the SLM-232 measured latent-state disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: append the SLM-231 measured recurrence-dynamics disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: append the SLM-230 measured recurrence disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: append the SLM-200 measured fixture disposition to the shared quality matrix without changing the SLM-197 scorer or experiment" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: append SLM-199 measured-results pointer to the shared quality matrix without changing the SLM-197 scorer or experiment" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: append SLM-198 measured-results pointer to the shared quality matrix without changing the SLM-197 scorer or experiment" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial registration; SLM-197 exact-candidate direct policy, set-mass loss, matched time encodings, checkpoint migration, free-running decode, and honest D0-D5 fixture matrix" + } + ] + }, + "harness.experiments.slm198_bridge_curriculum": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm198_bridge_curriculum.py", + "scripts/run_bridge_curriculum_matrix.py", + "tests/test_harnesses/experiments/test_slm198_bridge_curriculum.py", + "tests/test_scripts/test_run_bridge_curriculum_matrix.py", + "docs/design/iter-slm198-bridge-curriculum-20260723.json", + "docs/design/iter-slm198-bridge-curriculum-20260723.md", + "docs/design/iter-slm198-bridge-curriculum-agentv-20260723/" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "Deduplicate per-seed manifests, retain decode trace digests instead of repeated traces, and compact deployable evidence without changing measured metrics" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial registration; SLM-198 target-first bridge curriculum manifest, deterministic scheduling features, exposure proofs, resumable sampler, and honest fixture matrix" + } + ] + }, + "harness.experiments.slm199_legal_edit_flow": { + "version": "v4", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm199_legal_edit_flow.py", + "src/slm_training/models/legal_edit_flow.py", + "src/slm_training/flow/targets.py", + "src/slm_training/flow/samplers.py", + "scripts/run_legal_edit_flow_fixture.py", + "scripts/train_legal_edit_flow.py", + "tests/test_harnesses/experiments/test_slm199_legal_edit_flow.py", + "tests/test_models/test_legal_edit_flow.py", + "tests/test_flow/test_targets.py", + "tests/test_flow/test_samplers.py", + "tests/test_scripts/test_run_legal_edit_flow_fixture.py", + "tests/test_scripts/test_vercel_bundle_policy.py", + "vercel.json", + ".vercelignore", + "docs/design/iter-slm199-legal-edit-flow-20260723.json", + "docs/design/iter-slm199-legal-edit-flow-20260723.md", + "docs/design/iter-slm199-legal-edit-flow-agentv-20260723/" + ], + "history": [ + { + "version": "v4", + "date": "2026-07-24", + "note": "no-bump: exclude the offline SLM-233 campaign runner from Vercel source upload without changing deployed web behavior" + }, + { + "version": "v4", + "date": "2026-07-23", + "note": "move nested AgentV evidence exclusion to Vercel source-upload policy after live previews proved the function-level glob did not remove those files" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "exclude SLM-243 offline matrix code from the Vercel source upload while retaining its compact dashboard result JSON" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: replace the ineffective SLM-230 function glob with a source-upload ignore for the offline corpus without changing web runtime behavior" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: exclude SLM-230 offline corpus and evidence directories from the Vercel function bundle without changing web runtime behavior" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: generalize the offline Vercel harness exclusion to SLM-199/200 and exclude the SLM-200 top-level experiment report while preserving measured experiment behavior" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: regenerate SLM-199 evidence from clean v2 code with field-level AgentV checks and stamped result artifacts" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "enforce field-level AgentV checks, restricted checkpoint loading, contained certificate failures, and canonical shared run budgets" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: compact equivalent Vercel exclusions below the 256-character schema limit after preview validation rejected the expanded spelling" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: exclude offline-only SLM-199 train and experiment modules from the Vercel runtime bundle without changing rate, sampler, or evidence behavior" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial registration; SLM-199 exact finite-graph rate targets, UNKNOWN-safe adapted bridge targets, positive-rate model, exact Gillespie oracle, bounded production sampler, and honest fixture evidence" + } + ] + }, + "harness.experiments.slm200_flow_objective_attribution": { + "version": "v4", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm200_flow_objective_attribution.py", + "scripts/run_flow_objective_attribution.py", + "tests/test_harnesses/experiments/test_slm200_flow_objective_attribution.py", + "tests/test_scripts/test_run_flow_objective_attribution.py" + ], + "history": [ + { + "version": "v4", + "date": "2026-07-23", + "note": "no-bump: make the portable-path helper recognize its stable artifact-directory marker so the existing worktree-independent contract is testable on CI" + }, + { + "version": "v4", + "date": "2026-07-23", + "note": "rewrite AgentV artifact paths and encoded projection identities to portable agentv-dir references before committing evidence" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "restrict AgentV primary-parity checks to named booleans while retaining numeric parameter and initialization evidence in the result payload" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "fix normalized version-stamp construction in the SLM-200 publisher before measured evidence generation" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial registration; SLM-200 A0-A9 matched objective registry, exact-oracle control, parity validation, powered-protocol pins, confirmation firewall, and honest fixture-screen disposition" + } + ] + }, + "harness.experiments.slm209_debt_targeted_curriculum": { + "version": "v3", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm209_debt_targeted_curriculum.py", + "scripts/run_slm209_debt_targeted_curriculum_fixture.py", + "tests/test_harnesses/experiments/test_slm209_debt_targeted_curriculum.py", + "tests/test_scripts/test_run_slm209_debt_targeted_curriculum_fixture.py", + "docs/design/iter-slm209-debt-targeted-curriculum-20260720.json", + "docs/design/iter-slm209-debt-targeted-curriculum-20260720.md" + ], + "history": [ + { + "version": "v3", + "date": "2026-07-21", + "note": "extend owner with SLM-210 SDE5FloorEscapeMatrixV1, weight-permuted exposure control, and plan/mass axis audit" + }, + { + "version": "v2", + "date": "2026-07-20", + "note": "SLM-209 make debt digests deterministic (drop timestamp); add --design-json/--design-md overrides" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-209 SDE5-02 debt-targeted semantic exposure curriculum wiring/fixture harness" + } + ] + }, + "harness.experiments.slm211_untied_output_head": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm211_untied_output_head.py", + "src/slm_training/models/blocks.py", + "src/slm_training/models/hf_denoiser.py", + "src/slm_training/runtime/cactus/__init__.py", + "tests/test_models/test_tie_output_embedding.py", + "docs/design/iter-slm211-untied-output-head-20260721.json", + "docs/design/iter-slm211-untied-output-head-20260721.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-211 SDE5-04 default-on output-head tying control, copy-init untied arm, explicit checkpoint migration, optimizer deduplication, and spectral/Cactus metadata" + } + ] + }, + "harness.experiments.slm212_debt_routing": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm212_debt_routing.py", + "tests/test_harnesses/experiments/test_slm212_debt_routing.py", + "tests/test_scripts/test_run_slm212_debt_routing_fixture.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-212 SDE5-05 constraint-debt routing over decode paths wiring/fixture harness" + } + ] + }, + "harness.experiments.slm214_spectral_snapshot": { + "version": "v4", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm214_spectral_snapshot.py", + "scripts/inspect_spectral.py", + "tests/test_harnesses/experiments/test_slm214_spectral_snapshot.py", + "tests/test_scripts/test_inspect_spectral.py", + "docs/design/iter-slm214-spectral-snapshot-20260721.json", + "docs/design/iter-slm214-spectral-snapshot-20260721.md", + ".agents/skills/autotrain/references/inspect.md" + ], + "history": [ + { + "version": "v4", + "date": "2026-07-23", + "note": "expose canonical scale-invariant spectral trap projection for SLM-219 without a parallel SVD owner" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "make Xavier and Kaiming null draws deterministic with supported generator-aware uniform sampling" + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "no-bump: update test_inspect_spectral literal to match current component version v2; fixture/report semantics unchanged" + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "SLM-211: record tie_output_embedding in SpectralSnapshotV1" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-214 NCS0-01 SpectralSnapshotV1 native-PyTorch spectral backend, role registry, tied-storage detection, and randomized-ESD null calibration" + } + ] + }, + "harness.experiments.slm215_spectral_atlas": { + "version": "v3", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm215_spectral_atlas.py", + "scripts/build_spectral_atlas.py", + "tests/test_harnesses/experiments/test_slm215_spectral_atlas.py", + "tests/test_scripts/test_build_spectral_atlas.py", + "docs/design/iter-slm215-spectral-atlas-20260721.json", + "docs/design/iter-slm215-spectral-atlas-20260721.md" + ], + "history": [ + { + "version": "v3", + "date": "2026-07-23", + "note": "no-bump: pin the custom gate-path regression to the current honest verdict" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "no-bump: assert that custom gate references and hashes identify the same loaded artifact" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "record the actual caller-supplied SemanticFloorGateV1 path in atlas manifests" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "bind NCS atlas manifests to SemanticFloorGateV1 and keep strict meaning-v2 separate from legacy meaningful rate" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-215 NCS0-02 SpectralAtlasV1 fixture harness for null-calibrated checkpoint atlas and outcome joins" + } + ] + }, + "harness.experiments.slm216_spectral_regime": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm216_spectral_regime.py", + "scripts/run_spectral_regime_matrix.py", + "tests/test_harnesses/experiments/test_slm216_spectral_regime.py", + "tests/test_scripts/test_run_spectral_regime_matrix.py", + "docs/design/iter-slm216-spectral-regime-20260723.json", + "docs/design/iter-slm216-spectral-regime-20260723.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial SLM-216 fixed-token batch/data-scale matrix and SpectralRegimeGateV1" + } + ] + }, + "harness.experiments.slm226_absolute_spectral_gate": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm226_absolute_spectral_gate.py", + "scripts/run_absolute_spectral_gate.py", + "tests/test_harnesses/experiments/test_slm226_absolute_spectral_gate.py", + "tests/test_scripts/test_run_absolute_spectral_gate.py", + "docs/design/iter-slm226-absolute-spectral-gate-20260723.json", + "docs/design/iter-slm226-absolute-spectral-gate-20260723.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: refresh measured docs from the clean provenance-pinning implementation commit" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: pin check-mode provenance to the clean evidence commit across docs-only follow-up commits" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial finite-size null and scratch-width matrix plus AbsoluteSpectralTargetGateV1" + } + ] + }, + "harness.experiments.slm228_spectral_disposition": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm228_spectral_disposition.py", + "scripts/publish_spectral_disposition.py", + "scripts/model_cycle.py", + "tests/test_harnesses/experiments/test_slm228_spectral_disposition.py", + "tests/test_scripts/test_publish_spectral_disposition.py", + "docs/design/null-calibrated-spectral-learning-disposition.json", + "docs/design/null-calibrated-spectral-learning-disposition.md", + "docs/design/research-lineage.md", + "docs/MODEL_CARD.md", + "README.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: add the SLM-294 diagnostic checkpoint row to the shared model card without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: add SLM-248 research-intake contract to shared lineage without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: cross-link the SLM-236 blocked RSC4 disposition in shared README, model-card, and lineage docs without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: add SLM-194 adapted proposal-amortization evidence to shared research lineage without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: record the SLM-234 not_authorized gate closeout in shared research lineage without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: add the SLM-233 repository-diagnostic evidence to shared research lineage without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: add the SLM-243 adapted recursive-update evidence to shared research lineage without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: add the SLM-232 bounded latent-state evidence to shared roster and lineage docs without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: add the SLM-231 bounded recurrence-dynamics evidence to shared roster, operating reference, and lineage docs without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: add the SLM-230 rejected checkpoint and adapted recurrence evidence to shared roster and lineage docs without changing the SLM-228 disposition" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: refresh the generated disposition with its clean provenance-pinning implementation commit" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial final spectral disposition, evidence hashes, narrative, and promotion guards" + } + ] + }, + "harness.experiments.slm230_recurrence_observability": { + "version": "v12", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm230_recurrence_observability.py", + "scripts/run_slm230_recurrence_observability.py", + "tests/test_harnesses/experiments/test_slm230_recurrence_observability.py", + "tests/test_scripts/test_run_slm230_recurrence_observability.py", + "docs/design/iter-slm230-recurrence-observability-20260724.json", + "docs/design/iter-slm230-recurrence-observability-20260724.md", + "docs/design/iter-slm230-recurrence-observability-agentv-20260724/" + ], + "history": [ + { + "version": "v12", + "date": "2026-07-23", + "note": "run reproduction AgentV publishing in a temporary directory so check mode never rewrites committed evidence artifacts" + }, + { + "version": "v11", + "date": "2026-07-23", + "note": "no-bump: publish the clean reproducible report and pinned AgentV bundle for the already-versioned v11 harness" + }, + { + "version": "v11", + "date": "2026-07-23", + "note": "render AgentV summaries and policy rows in canonical key order so Markdown is stable across JSON serialization" + }, + { + "version": "v10", + "date": "2026-07-23", + "note": "preserve the committed dirty-evidence policy during reproduction instead of changing the hashed allow_dirty gate in check mode" + }, + { + "version": "v9", + "date": "2026-07-23", + "note": "seed every record-depth free-running decode independently so exact reproduction is invariant to preceding process RNG state" + }, + { + "version": "v8", + "date": "2026-07-23", + "note": "retain measured CPU latency in reports while excluding nondeterministic timing fields from exact scientific reproduction hashes" + }, + { + "version": "v7", + "date": "2026-07-23", + "note": "centralize the scientific report hash so generation and validation share runtime-noise exclusions while preserving the pinned evidence stamp" + }, + { + "version": "v6", + "date": "2026-07-23", + "note": "pin committed evidence stamps during reproduction and exclude elapsed runtime noise from the scientific report hash" + }, + { + "version": "v5", + "date": "2026-07-23", + "note": "count canonical recursive_outputs calls directly so every recurrence block evaluation is charged" + }, + { + "version": "v4", + "date": "2026-07-23", + "note": "bound free-running depth evaluation to two records per split and one hashed 32-token decoder policy under the hard run cap" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "bind heldout evaluation to the committed symbol-only suite and record the canonical three-minute run cap" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "bind the runner to the clean retry checkpoint and the measured 97-record strict corpus" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial RecurrenceObservabilityV1, RecurrenceExitPolicyV1, bounded checkpoint audit, and anytime controls" + } + ] + }, + "harness.experiments.slm217_functional_spectra": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm217_functional_spectra.py", + "scripts/run_functional_spectral_fixture.py", + "tests/test_harnesses/experiments/test_slm217_functional_spectra.py", + "tests/test_scripts/test_run_functional_spectral_fixture.py", + "docs/design/iter-slm217-functional-spectra-20260723.json", + "docs/design/iter-slm217-functional-spectra-20260723.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial FunctionalSpectralSnapshotV1 exact-state contract and bounded fixture" + } + ] + }, + "harness.experiments.slm218_cross_attention_retention": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm218_cross_attention_retention.py", + "scripts/run_cross_attention_retention.py", + "tests/test_harnesses/experiments/test_slm218_cross_attention_retention.py", + "tests/test_scripts/test_run_cross_attention_retention.py", + "docs/design/iter-slm218-cross-attention-retention-20260723.json", + "docs/design/iter-slm218-cross-attention-retention-20260723.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial CrossAttentionRetentionReportV1 geometry and fail-closed family coverage" + } + ] + }, + "harness.experiments.slm219_correlation_traps": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm219_correlation_traps.py", + "scripts/collect_correlation_trap_evidence.py", + "scripts/run_correlation_trap_retrospective.py", + "tests/test_harnesses/experiments/test_slm219_correlation_traps.py", + "tests/test_scripts/test_run_correlation_trap_retrospective.py", + "docs/design/iter-slm219-correlation-trap-evidence-20260723.json", + "docs/design/iter-slm219-correlation-trap-agentv-20260723/", + "docs/design/iter-slm219-correlation-trap-20260723.json", + "docs/design/iter-slm219-correlation-trap-20260723.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial CorrelationTrapReportV1 with deterministic six-checkpoint prefix evidence, independent collapse preregistration, confirmation-timed warnings, native and WeightWatcher controls, and fail-closed decision" + } + ] + }, + "harness.experiments.slm220_causal_subspace": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm220_causal_subspace.py", + "scripts/run_causal_subspace_fixture.py", + "tests/test_harnesses/experiments/test_slm220_causal_subspace.py", + "tests/test_scripts/test_run_causal_subspace_fixture.py", + "docs/design/iter-slm220-causal-subspace-20260723.json", + "docs/design/iter-slm220-causal-subspace-20260723.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial CausalSubspaceSnapshotV1 activation-side restriction estimator and rejected fixture-only retrospective" + } + ] + }, + "harness.experiments.slm222_muon_baseline": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm222_muon_baseline.py", + "docs/design/iter-slm222-muon-baseline-20260721.json", + "docs/design/iter-slm222-muon-baseline-20260721.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-222 NCS2-03 Muon/AdamW hybrid optimizer baseline fixture harness" + } + ] + }, + "harness.experiments.slm229_looped_latent_differentiation": { + "version": "v3", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm229_looped_latent_differentiation.py", + "scripts/run_slm229_looped_latent_differentiation.py", + "tests/test_harnesses/experiments/test_slm229_looped_latent_differentiation.py", + "tests/test_scripts/test_run_slm229_looped_latent_differentiation.py", + "docs/design/iter-slm229-looped-latent-differentiation-20260721.json", + "docs/design/iter-slm229-looped-latent-differentiation-20260721.md" + ], + "history": [ + { + "version": "v3", + "date": "2026-07-23", + "note": "derive learned-latent authorization narratives from the resolved semantic floor verdict" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: add explicit design output paths so fixture tests do not rewrite canonical docs; report semantics are unchanged" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "resolve the RSC learned-latent prerequisite through the authoritative SemanticFloorGateV1 hash" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "no-bump: regenerate fixture docs after merge-conflict resolution; harness code and matrix version unchanged" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-229 RSC0-01 zero-compute looped-latent differentiation and MinimalCompilerLatentContractV1 authorization audit" + } + ] + }, + "harness.gate_census": { + "version": "v8", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/model_build/evidence_census.py", + "scripts/census_ship_evidence.py", + "tests/test_harnesses/model_build/test_evidence_census.py" + ], + "history": [ + { + "version": "v8", + "date": "2026-07-23", + "note": "Preserve root metric-block JSON pointers and supersede their legacy normalized placeholders" + }, + { + "version": "v7", + "date": "2026-07-23", + "note": "Normalize legacy root and result scoreboard pointers when superseding preserved adjudication events" + }, + { + "version": "v6", + "date": "2026-07-23", + "note": "Report executing revision and canonical historical selection in the deterministic markdown renderer" + }, + { + "version": "v5", + "date": "2026-07-23", + "note": "Append a superseding adjudication when the replayed gate policy changes for an unchanged source scoreboard" + }, + { + "version": "v4", + "date": "2026-07-23", + "note": "Exclude prior ship-gate census artifacts from recursive self-census" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "Reuse canonical historical normalization, preserve verified ledger prefixes, stamp executing code, and compute exact-count interval overlap" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "Hash-chain append-only historical adjudications with deterministic supersession fields" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "Initial deterministic no-write committed-scoreboard census and append-only adjudication schema" + } + ] + }, + "harness.experiment_feature_flags": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/model_build/feature_flags.py", + "scripts/backfill_experiment_feature_flags.py", + "src/slm_training/resources/experiment_feature_flag_history.json" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "Persist experiment feature snapshots and historical evidence by canonical OpenFeature key" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "Initial OpenFeature-backed model-build lever registry, run snapshots, and historical evidence backfill" + } + ] + }, + "harness.experiments.slm231_recurrence_dynamics": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm231_recurrence_dynamics.py", + "scripts/run_slm231_recurrence_dynamics.py", + "tests/test_harnesses/experiments/test_slm231_recurrence_dynamics.py", + "tests/test_scripts/test_run_slm231_recurrence_dynamics.py", + "docs/design/iter-slm231-recurrence-dynamics-20260724.json", + "docs/design/iter-slm231-recurrence-dynamics-20260724.md", + "docs/design/iter-slm231-recurrence-dynamics-agentv-20260724/" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: publish the clean reproducible SLM-231 report and AgentV bundle generated against implementation commit 9d22aa81" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial residual-correct increment, composite, trajectory-product, FTLE, and cross-depth alignment diagnostics with synthetic, fixture, and bounded-checkpoint evidence" + } + ] + }, + "harness.experiments.slm232_latent_state_use": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm232_latent_state_use.py", + "scripts/run_slm232_latent_state_use.py", + "tests/test_harnesses/experiments/test_slm232_latent_state_use.py", + "tests/test_scripts/test_run_slm232_latent_state_use.py", + "docs/design/iter-slm232-latent-state-use-20260724.json", + "docs/design/iter-slm232-latent-state-use-20260724.md", + "docs/design/iter-slm232-latent-state-use-agentv-20260724/" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: publish the clean reproducible SLM-232 report and AgentV bundle generated against implementation commit 051c1c5d" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial bounded z-state rank, context decomposition, causal ablation, matched y-only/nuisance control, prior-join, and fail-closed disposition gate" + } + ] + }, + "harness.experiments.slm243_recursive_update_gate": { + "version": "v3", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm243_recursive_update_gate.py", + "scripts/run_slm243_recursive_update_gate.py", + "tests/test_harnesses/experiments/test_slm243_recursive_update_gate.py", + "tests/test_scripts/test_run_slm243_recursive_update_gate.py", + "docs/design/iter-slm243-recursive-update-gate-20260724.json", + "docs/design/iter-slm243-recursive-update-gate-20260724.md", + "docs/design/iter-slm243-recursive-update-gate-agentv-20260724/" + ], + "history": [ + { + "version": "v3", + "date": "2026-07-23", + "note": "no-bump: publish the compact dashboard report and hash-bound raw matrix generated cleanly against implementation commit 62726d66" + }, + { + "version": "v3", + "date": "2026-07-23", + "note": "split full 90-cell telemetry into a hash-bound raw matrix under the Vercel-excluded AgentV directory while retaining all gate inputs and semantic aggregates in dashboard JSON" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "deduplicate triangular depth telemetry and retain raw semantic rows at R=8 so dashboard evidence stays deployable without changing any gate input or verdict" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: publish the clean reproducible SLM-243 report and AgentV bundle generated against implementation commit c926cb05" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: make check mode replay the committed version stamp so durable evidence remains reproducible after its docs-only commit" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: serialize AgentV paths relative to the durable evidence directory so reports are machine-portable" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "no-bump: add deterministic mechanism fixtures proving historical empty-F pass-through, exact true-empty zero, near-identity scales and gates, and private norm identity" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial six-arm, five-depth, three-paired-seed recurrence update, empty-F, and private-norm architecture gate" + } + ] + }, + "harness.experiments.slm233_recursive_campaign": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm233_recursive_campaign.py", + "scripts/run_slm233_recursive_campaign.py", + "tests/test_harnesses/experiments/test_slm233_recursive_campaign.py", + "tests/test_scripts/test_run_slm233_recursive_campaign.py", + "docs/design/iter-slm233-recursive-campaign-20260724.json", + "docs/design/iter-slm233-recursive-campaign-20260724.md", + "docs/design/iter-slm233-recursive-campaign-agentv-20260724/" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-24", + "note": "no-bump: remove Markdown hard-break trailing spaces so repository diff validation remains clean" + }, + { + "version": "v2", + "date": "2026-07-24", + "note": "no-bump: make Markdown rendering stable across in-memory tuples and JSON-reloaded lists without changing scientific payloads" + }, + { + "version": "v2", + "date": "2026-07-24", + "note": "no-bump: publish the clean compact report, raw matrix, AgentV bundle, and measured-results narrative generated against implementation commit 6852e2cb" + }, + { + "version": "v2", + "date": "2026-07-24", + "note": "scope common initialization hash equality within each paired seed instead of incorrectly requiring different seed initializations to be identical" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "initial five-arm, three-paired-seed, gate-bounded matched recursive-depth campaign with fairness, exposure, test-depth, cost, resume, AgentV, and RecursiveCoreGateV2 evidence" + } + ] + }, + "harness.experiments.slm236_recurrent_latent_disposition": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm236_recurrent_latent_disposition.py", + "scripts/publish_recurrent_latent_disposition.py", + "tests/test_harnesses/experiments/test_slm236_recurrent_latent_disposition.py", + "tests/test_scripts/test_publish_recurrent_latent_disposition.py", + "docs/design/iter-slm236-recurrent-latent-disposition-20260724.json", + "docs/design/recurrent-semantic-computation-looped-latent-disposition.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial fail-closed RSC4 disposition assembled from merged gate evidence" + } + ] + }, + "harness.experiments.slm288_meaningful_validation": { + "version": "v1", + "kind": "harness", + "paths": [ + "scripts/run_slm288_meaningful_validation.py", + "tests/test_scripts/test_run_slm288_meaningful_validation.py", + "src/slm_training/resources/evals/slm288_annotation_packet_v1.json", + "src/slm_training/resources/evals/slm288_meaningful_rubric_v1.md", + "docs/design/iter-slm288-meaningful-validation-20260724.json", + "docs/design/iter-slm288-meaningful-validation-20260724.md", + "docs/design/slm288-annotation-labels-20260724.json" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-288 reason-code histograms, frozen blinded annotation packet, agent-rater agreement analysis, and metric disposition" + } + ] + }, + "harness.experiments.slm299_edit_reachability": { + "version": "v4", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm299_edit_reachability.py", + "scripts/run_slm299_reachability_audit.py", + "tests/test_harnesses/experiments/test_slm299_edit_reachability.py", + "docs/design/iter-slm299-edit-reachability-20260724.json", + "docs/design/iter-slm299-edit-reachability-20260724.md", + "src/slm_training/models/tree_edit_diffusion.py", + "docs/design/iter-slm305-edit-language-20260724.json", + "docs/design/iter-slm305-edit-language-20260724.md", + "tests/test_models/test_tree_edit_diffusion.py" + ], + "history": [ + { + "version": "v4", + "date": "2026-07-24", + "note": "SLM-310: tree_edit_diffusion gains reason-coded apply rejections, per-proposal decode telemetry (applicability/rejection/budget), corruption_action_distribution + stop_slot_accounting config knobs (defaults = historical behavior, checkpoint parity); test_tree_edit_diffusion unchanged semantics" + }, + { + "version": "v3", + "date": "2026-07-24", + "note": "SLM-308: tree_edit_diffusion gains value_label_mode (bounded_distance default for new trainings, mutation_count parity for pre-field checkpoints), oracle-backed normalized cost-to-go value targets with UNKNOWN exclusion, and pairwise parent/improving-child progress loss; edit space and reachability machinery unchanged" + }, + { + "version": "v2", + "date": "2026-07-24", + "note": "SLM-305: extend the valid-state edit language (ADD_CONTAINER/REMOVE_CONTAINER, INSERT/REPLACE_SUBTREE, V0.5 statement edits, BIND_PLACEHOLDER) with inverses, parser-backed validation, checkpoint format 2 + migration, and extended reachability mode" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; real X22 tree-edit reachability analyzer, suite audit CLI, and reachability-unproven gate input" + } + ] + }, + "harness.experiments.slm301_prompt_observability": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm301_prompt_observability.py", + "scripts/run_slm301_prompt_observability.py", + "tests/test_harnesses/experiments/test_slm301_prompt_observability.py", + "docs/design/iter-slm301-prompt-observability-20260724.json", + "docs/design/iter-slm301-prompt-observability-20260724.md", + "docs/design/slm301-slot-observability-audit-20260724.json" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-301 LAR1-04 prompt-observability arms (as-is vs +inventory) with phase-0 coverage classification, AR/X22 baselines, paired predeclared-delta report" + } + ] + }, + "harness.experiments.slm303_decode_budget_audit": { + "version": "v1", + "kind": "harness", + "paths": [ + "scripts/run_slm303_decode_budget_audit.py", + "tests/test_harnesses/model_build/test_decode_outcome.py", + "tests/test_scripts/test_run_slm303_decode_budget_audit.py", + "docs/design/iter-slm303-decode-budget-audit-20260724.json", + "docs/design/iter-slm303-decode-budget-audit-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-303 decode-budget audit with hash pins, preregistered budget sweep, and taxonomy disposition" + } + ] + }, + "harness.experiments.slm308_distance_value": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm308_distance_oracle.py", + "scripts/run_slm308_distance_value.py", + "tests/test_harnesses/experiments/test_slm308_distance_value.py", + "docs/design/iter-slm308-distance-value-20260724.json", + "docs/design/iter-slm308-distance-value-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-308 LAR2-02 bounded reverse-BFS distance oracle (EXACT/BOUNDED/UNKNOWN) over canonical AST fingerprints, distance value supervision + pairwise progress ranking in the X22 tree-edit model, matched fixture experiment with preregistered thresholds" + } + ] + }, + "harness.model_build.eval": { + "version": "v54", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/model_build/debt_router.py", + "src/slm_training/harnesses/model_build/eval_runner.py", + "src/slm_training/harnesses/model_build/eval_policy.py", + "src/slm_training/models/decode_stats.py", + "src/slm_training/harnesses/quality/retrieval.py", + "scripts/evaluate_model.py" + ], + "history": [ + { + "version": "v54", + "date": "2026-07-24", + "note": "Add bounded opt-in LangSmith aggregate telemetry to the shared evaluation trace while keeping local evidence authoritative" + }, + { + "version": "v53", + "date": "2026-07-24", + "note": "Add SLM-303 per-record decode outcome taxonomy and suite outcome counts without changing existing gate inputs" + }, + { + "version": "v52", + "date": "2026-07-24", + "note": "Add typed meaningful-program v1 reason reports while preserving the existing verdict and legacy reason strings" + }, + { + "version": "v51", + "date": "2026-07-24", + "note": "Publish AgentEvals-authoritative criteria with every policy evaluation and route CLI ship outcomes through that evidence" + }, + { + "version": "v50", + "date": "2026-07-24", + "note": "SLM-342: overlay explicit OpenFeature research-lever assignments on evaluation config and persist them in emitted results" + }, + { + "version": "v49", + "date": "2026-07-24", + "note": "Persist suite harness provenance and explicit raw, constrained, and repaired lineage IDs on every evaluation detail" + }, + { + "version": "v48", + "date": "2026-07-23", + "note": "Resolve and persist OpenFeature evaluation-lever snapshots in scoreboards" + }, + { + "version": "v47", + "date": "2026-07-23", + "note": "Keep power utilities off the import path, complete exact-match evidence, and represent non-binomial decode telemetry without invalid Wilson counts" + }, + { + "version": "v46", + "date": "2026-07-23", + "note": "Attach numerator, denominator, seed count, Wilson interval, and evidence class to canonical binomial rates" + }, + { + "version": "v45", + "date": "2026-07-22", + "note": "Count certified finalization fallbacks in decode telemetry and honest fallback gates" + }, + { + "version": "v44", + "date": "2026-07-22", + "note": "Add provenance-stamped eval offsets to sample non-prefix records locally without cache collisions" + }, + { + "version": "v43", + "date": "2026-07-22", + "note": "Include the canonical lever-contract version in eval cache identity and every suite or scoreboard version stamp" + }, + { + "version": "v42", + "date": "2026-07-22", + "note": "Persist the complete effective post-checkpoint model configuration in every evaluation policy and cache identity" + }, + { + "version": "v41", + "date": "2026-07-22", + "note": "Persist every ModelBuildConfig decode weight from the canonical field registry instead of a partial hand-maintained list" + }, + { + "version": "v40", + "date": "2026-07-21", + "note": "Keep atomic strict decode policy from mutating checkpoint schema, slot-contract, or design input conditioning" + }, + { + "version": "v39", + "date": "2026-07-21", + "note": "Normalize compiler-tree evaluation to one atomic strict policy before artifacts are created" + }, + { + "version": "v38", + "date": "2026-07-21", + "note": "Make free-form output strings an explicit meaningful_program_v1 failure" + }, + { + "version": "v37", + "date": "2026-07-21", + "note": "declare legacy visible slot-suffix semantic roles as typed runtime-symbol authority at the canonical evaluation request boundary" + }, + { + "version": "v36", + "date": "2026-07-21", + "note": "Merge E631 meaningful-program evaluation tightening with E675 and E687 evaluation policy and trace persistence" + }, + { + "version": "v35", + "date": "2026-07-21", + "note": "E687: attach stable evaluation record IDs to per-call constrained-selection traces before aggregation" + }, + { + "version": "v34", + "date": "2026-07-21", + "note": "E675: expose default-off schema open-property decode margin in evaluation policy and evidence" + }, + { + "version": "v34", + "date": "2026-07-21", + "note": "E631: _is_meaningful_program (meaningful_program_v1) gains an AST-structural empty-children check (_first_empty_children_component, walking Program.root) alongside its existing Stack([])/Card([]) literal substring checks -- purely additive, never removes an existing rejection. Fixes the gap E630 flagged and deferred: a component whose children array is genuinely empty but whose remaining non-content positional arguments got padded with unrelated stuffed values (e.g. `Card([], \":a\", \":b\")`) never matched the literal \"Card([])\" substring, so it passed as meaningful when it should not have. The AST check reads children from the parsed props dict, which is structurally separate from a node's other props, so padding those other props cannot hide an empty children array. Generalizes for free to any schema component the parser itself marks with a children prop (Modal, Carousel), which the old literal check never covered at all (new reason code `empty_children:`; Stack/Card keep their original `empty_root_stack`/`empty_card` reason strings for backward compatibility). Verified against the real scratch checkpoint reused from E626/E628/E630 (sha256 c5b7c807...dd561221): re-scoring the already-recorded rico_held suite at required_slot_margin_decode_weight=0 (pure control, no lever active) flips rico_eval_test_42 and rico_eval_test_77 from previously (incorrectly) True to correctly False (both have a `Card([], \":cardN.title\")`-shaped padded-empty Card already present in the checkpoint's baseline decode, unrelated to the lever) -- an honest, expected tightening, not a regression. New/updated tests in tests/test_evals/test_meaningful_program.py. See docs/design iter-e631." + }, + { + "version": "v33", + "date": "2026-07-20", + "note": "E626: expose and persist the default-off required-slot margin decode weight" + }, + { + "version": "v33", + "date": "2026-07-21", + "note": "SLM-212: add default-off constraint-debt routing config, DebtRoutingPolicy, CalibratedDebtRouter, CLI flags, and decode identity plumbing" + }, + { + "version": "v32", + "date": "2026-07-20", + "note": "no-bump: describe E633's model-side pre-content string routing in existing schema-opaque CLI help without changing evaluation plumbing" + }, + { + "version": "v32", + "date": "2026-07-20", + "note": "no-bump: restore the original schema-opaque CLI description when rejecting E632 without changing evaluation plumbing" + }, + { + "version": "v32", + "date": "2026-07-20", + "note": "no-bump: align schema-opaque CLI help with the expanded model-side non-content string policy without changing evaluation plumbing" + }, + { + "version": "v32", + "date": "2026-07-20", + "note": "no-bump: describe the expanded model-side slot coverage closure policy in CLI help without changing evaluation plumbing" + }, + { + "version": "v32", + "date": "2026-07-20", + "note": "expose and persist the default-off schema-derived typed-array item margin" + }, + { + "version": "v31", + "date": "2026-07-20", + "note": "expose and persist the default-off authored typed-array nonempty margin" + }, + { + "version": "v30", + "date": "2026-07-20", + "note": "expose and persist the default-off repeated-plan distinct-slot margin" + }, + { + "version": "v29", + "date": "2026-07-20", + "note": "expose and persist the default-off repeated-plan-family nested-array close margin" + }, + { + "version": "v28", + "date": "2026-07-20", + "note": "make the reported evaluation canvas cap govern request-aware generation length" + }, + { + "version": "v27", + "date": "2026-07-20", + "note": "expose and persist the default-off verifier-approved semantic-plan root margin" + }, + { + "version": "v26", + "date": "2026-07-20", + "note": "expose and persist the default-off semantic-plan legal-component margin score" + }, + { + "version": "v25", + "date": "2026-07-20", + "note": "expose the default-off first-component semantic-plan seed score in canonical evaluation policy" + }, + { + "version": "v24", + "date": "2026-07-20", + "note": "persist the default-off visible-slot coverage close score in canonical evaluation policy" + }, + { + "version": "v23", + "date": "2026-07-20", + "note": "persist the default-off public-schema semantic-role candidate policy" + }, + { + "version": "v22", + "date": "2026-07-20", + "note": "expose the default-off inline semantic-plan family score and persist it in canonical evaluation policy" + }, + { + "version": "v21", + "date": "2026-07-20", + "note": "expose the default-off enum-argument close score and persist it in canonical evaluation policy" + }, + { + "version": "v20", + "date": "2026-07-20", + "note": "expose the default-off schema-role slot score and persist it in canonical evaluation policy" + }, + { + "version": "v19", + "date": "2026-07-20", + "note": "expose the default-off optional opaque-argument close score and persist it in canonical evaluation policy" + }, + { + "version": "v18", + "date": "2026-07-20", + "note": "expose the default-off schema-opaque decode weight and persist it in canonical evaluation policy" + }, + { + "version": "v17", + "date": "2026-07-20", + "note": "expose the default-off schema-value decode weight and persist it in canonical evaluation policy" + }, + { + "version": "v16", + "date": "2026-07-20", + "note": "expose plan-coverage-gated legal Stack root construction and termination scoring in canonical evaluation" + }, + { + "version": "v15", + "date": "2026-07-20", + "note": "expose prompt-plan-compatible terminal-root reference soft scoring and its bounded intervention telemetry" + }, + { + "version": "v14", + "date": "2026-07-20", + "note": "expose prompt-derived SemanticPlanV1 soft scoring for legal choice-codec root and bound component decisions and stamp the TwoTower revision in its eval results" + }, + { + "version": "v13", + "date": "2026-07-19", + "note": "expose learned terminal-root reference identity bias and intervention telemetry in canonical evaluation" + }, + { + "version": "v12", + "date": "2026-07-19", + "note": "bound learned root-reference arity partitioning by the available generated sections so impossible tokenizer-tail classes cannot force continuation" + }, + { + "version": "v11", + "date": "2026-07-19", + "note": "add learned choice-codec terminal-root reference arity decoding and bounded intervention telemetry" + }, + { + "version": "v10", + "date": "2026-07-19", + "note": "restrict structural generated-reference completeness bias to the terminal root list" + }, + { + "version": "v9", + "date": "2026-07-19", + "note": "persist bounded root-vs-nested choice-frame and reference-bias counterfactual telemetry" + }, + { + "version": "v8", + "date": "2026-07-19", + "note": "extend generated-reference completeness bias to reachable structural list aggregation while remaining fail-closed elsewhere" + }, + { + "version": "v7", + "date": "2026-07-19", + "note": "persist bounded choice-token and legal-reference decision evidence for per-record eval diagnosis" + }, + { + "version": "v6", + "date": "2026-07-19", + "note": "add a fail-closed choice-codec bias toward unused legal generated bound references in the root" + }, + { + "version": "v5", + "date": "2026-07-19", + "note": "add a direct constrained-decode bias from honest visible semantic-role candidates" + }, + { + "version": "v4", + "date": "2026-07-19", + "note": "add honest inference-time semantic-role contract projection from prompt-mentioned components and visible slots" + }, + { + "version": "v3", + "date": "2026-07-19", + "note": "merge SDE3-01 suite-level eval cache integration on top of decode-weight override CLI" + }, + { + "version": "v2", + "date": "2026-07-19", + "note": "add SDE3-01 suite-level content-addressed eval cache integration (EvalCache, --eval-cache-mode, --eval-cache-root)" + }, + { + "version": "v2", + "date": "2026-07-19", + "note": "expose the existing slot-to-component decode-weight override in the canonical evaluation CLI" + }, + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration" + } + ] + }, + "harness.model_build.data": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/model_build/data.py" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "reject malformed or metadata-divergent Harness DSL prompts before train or eval records reach model input construction" + }, + { + "version": "v1", + "date": "2026-07-22", + "note": "Enforce the symbol-only completion contract in shared train and eval record loading before model construction or artifacts" + } + ] + }, + "harness.model_build.train": { + "version": "v24", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/model_build/train_loop.py", + "src/slm_training/harnesses/model_build/full_state.py" + ], + "history": [ + { + "version": "v24", + "date": "2026-07-23", + "note": "Reject disposition-forbidden spectral controls before direct ship evaluation or promoted training" + }, + { + "version": "v23", + "date": "2026-07-23", + "note": "Fail staged training before corpus or model initialization when capability certificates or lever permissions are invalid" + }, + { + "version": "v22", + "date": "2026-07-23", + "note": "Persist resolved OpenFeature training-lever snapshots" + }, + { + "version": "v21", + "date": "2026-07-23", + "note": "Fail before training when explicit checkpoint promotion lacks valid campaign governance evidence" + }, + { + "version": "v20", + "date": "2026-07-22", + "note": "Restore the root-reference identity strict-subset sampler to its reachable choice-tokenizer capability" + }, + { + "version": "v19", + "date": "2026-07-22", + "note": "Select strict-subset root-reference identity records through either choice or lexer grammar-native targets" + }, + { + "version": "v18", + "date": "2026-07-21", + "note": "no-bump: source the already-v18 two-minute training cap from the canonical lever registry; runtime behavior is unchanged" + }, + { + "version": "v18", + "date": "2026-07-21", + "note": "reduce the cumulative training deadline default and maximum from three minutes to two" + }, + { + "version": "v17", + "date": "2026-07-21", + "note": "SLM-222: add Muon/AdamW hybrid optimizer support with optimizer-family fingerprint and fail-closed resume guard; default AdamW path preserved" + }, + { + "version": "v16", + "date": "2026-07-20", + "note": "wire SDE2-03 exposure-targeted rare-action sampler kwargs and recipe metadata; default with_replacement sampler unchanged" + }, + { + "version": "v15", + "date": "2026-07-20", + "note": "add deterministic rare slot-owner record oversampling with auditable corpus counts and expanded exposure" + }, + { + "version": "v14", + "date": "2026-07-20", + "note": "persist slot next-context and pair-interaction architecture flags in train recipe summaries" + }, + { + "version": "v13", + "date": "2026-07-20", + "note": "rebuild deterministic corpus-derived slot priors after weight initialization instead of silently restoring stale parent checkpoint statistics" + }, + { + "version": "v12", + "date": "2026-07-19", + "note": "add deterministic strict-subset root-reference identity oversampling with auditable eligible and expanded sampling counts" + }, + { + "version": "v11", + "date": "2026-07-19", + "note": "record configurable excluded-reference loss weighting and negative-class calibration telemetry for terminal-root identity training" + }, + { + "version": "v10", + "date": "2026-07-19", + "note": "record learned terminal-root reference identity loss and decode settings in training recipes" + }, + { + "version": "v9", + "date": "2026-07-19", + "note": "mask impossible root-reference arity classes by each target stream's available generated sections during auxiliary training" + }, + { + "version": "v8", + "date": "2026-07-19", + "note": "train an isolated choice-codec terminal-root reference arity head and record its recipe settings" + }, + { + "version": "v7", + "date": "2026-07-19", + "note": "no-bump: isolate the train.progress heartbeat behind a warn-only guard so telemetry write failures can never abort training" + }, + { + "version": "v7", + "date": "2026-07-19", + "note": "no-bump: throttled OTLP train.progress heartbeat logs for live peer streaming; training math, metrics.jsonl rows, and summaries unchanged" + }, + { + "version": "v7", + "date": "2026-07-19", + "note": "expose honest prompt-derived slot-contract training and record contract provenance in train summaries" + }, + { + "version": "v6", + "date": "2026-07-19", + "note": "add bounded source-stratified masked-token loss proxy summaries for parent-replay attribution" + }, + { + "version": "v5", + "date": "2026-07-19", + "note": "add deterministic hash-pinned parent-corpus replay with fail-closed sampler composition and effective-exposure telemetry" + }, + { + "version": "v4", + "date": "2026-07-19", + "note": "add explicit initialized-weight retention with strict initialization guards and RMS drift telemetry" + }, + { + "version": "v3", + "date": "2026-07-18", + "note": "preserve corpus-derived serving priors during checkpoint initialization and expose restored fields plus slot-head recipe settings in train summaries" + }, + { + "version": "v2", + "date": "2026-07-18", + "note": "add explicit weight-only checkpoint initialization for matched new-corpus training without weakening bit-exact resume guards" + }, + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration" + } + ] + }, + "harness.oracle_scoring_replay": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/evals/oracle_scoring_replay.py", + "scripts/audit_gold_scoring.py", + "tests/test_evals/test_oracle_scoring_replay.py", + "docs/design/oracle-scoring-replay.md", + "docs/design/iter-slm260-oracle-scoring-replay-20260721.json", + "docs/design/iter-slm260-oracle-scoring-replay-20260724.json" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "no-bump: record a local 66-row oracle replay under unchanged scorer and harness behavior" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-260 VSD0-01 oracle scoring replay harness and audit CLI" + } + ] + }, + "harness.eval.semantic_failure_census": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/evals/semantic_failure.py", + "scripts/audit_semantic_failures.py", + "tests/test_evals/test_semantic_failure.py", + "tests/test_scripts/test_audit_semantic_failures.py", + "docs/design/semantic-failure-census.md", + "docs/design/iter-slm263-semantic-failure-census-20260724.json" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-263 deterministic first-semantic-failure taxonomy and immutable-generation census" + } + ] + }, + "harness.experiments.slm265_domain_shift_audit": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm265_domain_shift_audit.py", + "scripts/audit_domain_shift.py", + "tests/test_harnesses/experiments/test_slm265_domain_shift_audit.py", + "docs/design/rico-domain-shift-audit.md", + "docs/design/iter-slm265-domain-shift-audit-20260724.json" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-265 outcome-blind domain-shift strata and coverage-gap audit" + } + ] + }, + "harness.experiments.slm266_teacher_programs": { + "version": "v6", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/teacher_programs.py", + "scripts/build_teacher_program_generation_manifest.py", + "scripts/admit_teacher_programs.py", + "scripts/generate_teacher_programs.py", + "scripts/materialize_teacher_programs.py", + "tests/test_harnesses/experiments/test_teacher_programs.py", + "docs/design/teacher-program-data.md" + ], + "history": [ + { + "version": "v6", + "date": "2026-07-24", + "note": "add revision-pinned local-only Transformers teacher transport and CLI execution mode" + }, + { + "version": "v5", + "date": "2026-07-24", + "note": "bind deep teacher admission G11 to persisted cross-family automatic judge evidence" + }, + { + "version": "v4", + "date": "2026-07-24", + "note": "execute locked teacher requests through budgeted provider transport with content-addressed raw attempt archive" + }, + { + "version": "v3", + "date": "2026-07-24", + "note": "materialize only deep-admitted teacher rows as canonical train snapshots with immutable raw archive pointer" + }, + { + "version": "v2", + "date": "2026-07-24", + "note": "make G12 optional evidence and derive Silver admission only from automated deep verification" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-266 fail-closed teacher-program request and offline admission contracts" + } + ] + }, + "harness.eval.semantic_trajectory": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/evals/semantic_trajectory.py", + "scripts/collect_semantic_trajectory.py", + "tests/test_evals/test_semantic_trajectory.py", + "src/slm_training/resources/evals/slm264_trajectory_fixture.jsonl", + "docs/design/semantic-collapse-telemetry.md", + "docs/design/iter-slm264-semantic-collapse-telemetry-20260724.json" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-264 default-off semantic trajectory schema and evidence inventory" + } + ] + }, + "harness.preference.constraint_debt": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/preference/constraint_debt.py", + "src/slm_training/harnesses/preference/local_train.py", + "scripts/diagnose_preference_gradient_alignment.py", + "tests/test_harnesses/preference/test_constraint_debt.py", + "tests/test_harnesses/preference/test_local_train_constraint_debt.py", + "docs/design/iter-slm208-constraint-debt-20260720.json", + "docs/design/iter-slm208-constraint-debt-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: SLM-209 reuses ConstraintDebtV1 rows and compute_constraint_debt_v1 types only; no behavior changed" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-208 SDE5-01 constraint-debt V1 telemetry and pre/post-mask grammar-distortion KL wiring" + } + ] + }, + "harness.preference.remine_campaign": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/preference/remine_campaign.py", + "scripts/run_remine_campaign.py", + "scripts/autoresearch.py", + "tests/test_harnesses/preference/test_remine_campaign.py", + "tests/test_scripts/test_autoresearch_remine.py", + "docs/design/ldi3-04-remine-intervene-campaign-20260718.md", + "docs/design/ldi3-04-remine-intervene-campaign-report-20260718.json" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: scripts/autoresearch.py adds independent campaign-lock enforcement; remine campaign behavior is unchanged" + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "no-bump: source the already-v2 remine and autoresearch cap from the canonical lever registry; runtime behavior is unchanged" + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "reduce remine and autoresearch campaign wall-budget default and maximum to two minutes" + }, + { + "version": "v1", + "date": "2026-07-21", + "note": "no-bump: scripts/autoresearch.py gains an export-openfeature subcommand (flagd flag-definition export of a hypothesis matrix); remine campaign behavior unchanged" + }, + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration; SLM-132 LDI3-04 immutable on-policy remine/intervene/regenerate campaign wiring/fixture harness" + } + ] + }, + "harness.quality.slop_forensics": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/quality/slop_forensics.py", + "scripts/run_slop_forensics.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration" + } + ] + }, + "harness.reasoning.bench": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/reasoning/" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration" + } + ] + }, + "harness.representations": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/representations/", + "scripts/run_sae_diagnostic_fixture.py", + "tests/test_harnesses/representations/", + "tests/test_scripts/test_run_sae_diagnostic_fixture.py", + "docs/design/iter-ldi4-02-sae-decision-state-diagnostic-20260720.json", + "docs/design/iter-ldi4-02-sae-decision-state-diagnostic-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact timestamp after fixture CLI rerun; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration of SLM-136 LDI4-02 SAE decision-state diagnostic harness (schema, SAE, interventions, fixture wiring)" + } + ] + }, + "harness.capability_artifacts": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/capability_artifacts.py", + "tests/test_harnesses/test_capability_artifacts.py", + "docs/design/dsh0-04-capability-artifacts-20260723.json", + "docs/design/dsh0-04-capability-artifacts-20260723.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial content-addressed answer, question, QA-set, derivation, validation, and capability-certificate schemas with publication lineage checks" + } + ] + }, + "harness.capability_gates": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/capability_gates.py", + "scripts/manage_capability_certificate.py", + "tests/test_harnesses/test_capability_gates.py", + "docs/design/dsh0-07-capability-gates-20260723.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "Add immutable capability gate specs, results, certificates, retention and prerequisite enforcement, and explicit promotion commands" + } + ] + }, + "harness.artifact_graph": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/train_data/artifact_graph.py", + "src/slm_training/harnesses/train_data/split_policy.py", + "scripts/explain_artifact_overlaps.py", + "tests/test_harnesses/train_data/test_artifact_graph.py", + "docs/design/dsh0-05-artifact-graph-20260723.json", + "docs/design/dsh0-05-artifact-graph-20260723.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial append-only dataset artifact graph, root-family split inheritance, cross-split leakage quarantine, traversal, and explain CLI" + } + ] + }, + "harness.solver_bench": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/solver_bench.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration" + } + ] + }, + "harness.staged": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/staged.py", + "tests/test_harnesses/test_staged.py", + "scripts/verify_foundation_disposition.py", + "docs/design/dsh0-01-staged-harness-baseline-20260723.json", + "docs/design/dsh0-01-staged-harness-baseline-20260723.md", + "docs/design/dsh0-02-symbolic-surface-policy-20260723.json", + "docs/design/dsh0-02-symbolic-surface-policy-20260723.md", + "docs/design/dsh0-08-foundation-disposition-20260723.json", + "docs/design/dsh0-08-foundation-disposition-20260723.md" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: refresh pinned pack, CAP0 plan, materialization, and disposition hashes after adding the independent Harness DSL boundary" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: refresh pinned DslPack hashes after adding independent minimal-witness authority" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: refresh pinned DslPack source hashes after adding the independent grammar capability authority slot" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "Add the typed fail-closed G0 foundation disposition, current-artifact verification, and exact CAP0 identity freeze" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial staged DSL capability taxonomy and fail-closed StagedHarnessBaselineV1 evidence identity contract" + } + ] + }, + "harness.synthesis_plan": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/synthesis_plan.py", + "src/slm_training/resources/synthesis_plans/", + "tests/test_harnesses/test_synthesis_plan.py", + "docs/design/dsh0-03-synthesis-plan-20260723.json", + "docs/design/dsh0-03-synthesis-plan-20260723.md" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "no-bump: refresh the executable CAP0 fixture to the v18 symbolic task materialization contract" + }, + { + "version": "v2", + "date": "2026-07-23", + "note": "refresh the executable CAP0 fixture to the v17 train-data materialization contract and preserve its prior identity as historical evidence" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial checked-in SynthesisPlanV1, pack-authoritative component preflight, and executable capability transition state machine" + } + ] + }, + "harness.train_data": { + "version": "v19", + "kind": "harness", + "paths": [ + "scripts/build_train_data.py", + "src/slm_training/harnesses/train_data/", + "src/slm_training/data/language_contract/", + "src/slm_training/dsl/renderability.py", + "src/slm_training/data/mixture.py", + "src/slm_training/dsl/packs/openui.py", + "src/slm_training/resources/train_seeds.jsonl", + "src/slm_training/resources/frontier/" + ], + "history": [ + { + "version": "v19", + "date": "2026-07-23", + "note": "add runtime-renderability repairs and chosen-over-rejected preference pairs for structural-only OpenUI roots" + }, + { + "version": "v18", + "date": "2026-07-23", + "note": "emit identity and canonical CAP0 scope rows as validated symbolic Harness DSL while preserving legacy repair and typed-map semantics" + }, + { + "version": "v17", + "date": "2026-07-23", + "note": "materialize synthesis-plan artifact graphs through canonical ExampleRecord, validation, preference, quality, leakage, and publication paths" + }, + { + "version": "v16", + "date": "2026-07-23", + "note": "add staged dataset artifact-graph sidecars and root-family cross-split leakage firewall" + }, + { + "version": "v15", + "date": "2026-07-23", + "note": "validate optional checked-in SynthesisPlanV1 before loading any train-data producer and record its exact identity in the manifest" + }, + { + "version": "v14", + "date": "2026-07-23", + "note": "remove the train_text_only_01 source fixture after strict sanitization proved it structurally duplicates the reserved adv_empty_prompt_01 test fallback" + }, + { + "version": "v13", + "date": "2026-07-23", + "note": "emit symbol-only collapsed multi-instruction operator records with exact trace and hard-negative evidence" + }, + { + "version": "v12", + "date": "2026-07-23", + "note": "no-bump: pin operator contract and legal-set component versions in the sibling report stamp; generated corpus behavior is unchanged" + }, + { + "version": "v12", + "date": "2026-07-23", + "note": "generate replay-verified symbolic single-turn, next-turn, and sibling-fork operator datasets from admitted exact AST roots" + }, + { + "version": "v11", + "date": "2026-07-22", + "note": "Templatize every OpenUI pack-generator target before record emission and remove stale free-form settings names from canonical train seeds" + }, + { + "version": "v10", + "date": "2026-07-21", + "note": "Fail closed unless sanitization has templatized every non-grammar string, including arrays and operational string fields" + }, + { + "version": "v9", + "date": "2026-07-20", + "note": "add SDE2-03 exposure-targeted rare-action online sampler with bounded importance weights and diversity caps; default with_replacement unchanged" + }, + { + "version": "v8", + "date": "2026-07-19", + "note": "add deterministic target sanitization (D2 canonicalization + schema-checked default-elision/dead-binding removal/guarded Stack flattening + content-literal templatization) behind sanitize_mode off/audit/enforce; strict profile defaults to enforce" + }, + { + "version": "v7", + "date": "2026-07-19", + "note": "no-bump: reuse the semantic-role formatter from the shared quality owner without changing projected training records" + }, + { + "version": "v7", + "date": "2026-07-19", + "note": "add an opt-in visible semantic-role contract derived only from visible slots, visible component types, and generated schema compatibility" + }, + { + "version": "v6", + "date": "2026-07-19", + "note": "support a type-only visible component prompt contract for matched count-ablation data" + }, + { + "version": "v5", + "date": "2026-07-19", + "note": "add diversity fingerprinting module (diversity.py), corpus audit script, and regression tests for canonical-root, topology, type/action, prompt-intent, and lineage diversity metrics" + }, + { + "version": "v4", + "date": "2026-07-19", + "note": "add an opt-in post-admission visible component type/count contract to training prompts" + }, + { + "version": "v3", + "date": "2026-07-19", + "note": "add synthetic-data integrity gate module (integrity.py), audit script, and regression tests for canonical AST, codec round-trip, binding graph, slot contract, and leakage checks" + }, + { + "version": "v2", + "date": "2026-07-18", + "note": "add provenance-tagged expression-to-document projection and explicit target-kind corpus selection" + }, + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration; strict build results carry version stamps and actionable rejection lineage" + } + ] + }, + "harness_core.lineage.interventions": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harness_core/lineage/interventions.py", + "scripts/run_intervention_unification_fixture.py", + "tests/test_lineage_interventions.py", + "tests/test_scripts/test_run_intervention_unification_fixture.py", + "docs/design/iter-ldi4-03-intervention-unification-20260720.json", + "docs/design/iter-ldi4-03-intervention-unification-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: regenerated design doc after harness.experiments v50 stamp bump; no harness behavior change" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after sibling fixture test run; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v42 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments v41 bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact timestamp/version-stamp refreshed after harness.experiments bump; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refreshed design artifact timestamps/metrics only; harness behavior and thresholds unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: design artifact re-stamped after harness.experiments v35 bump; timestamp/version_stamp metadata only" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps after harness.experiments v31 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: refresh design artifact version stamps and trim trailing whitespace after harness.experiments v29 bump; report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration of SLM-137 LDI4-03 unified intervention manifest/registry/promotion harness" + } + ] + }, + "matrix.b3_capacity_v2": { + "version": "v1", + "kind": "matrix", + "paths": [ + "scripts/run_b3_capacity_v2.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration; SLM-124 EFS3-03 B3 surface-vs-choice capacity-ladder v2 plan/fixture CLI" + } + ] + }, + "matrix.causal_peft_ftpo": { + "version": "v1", + "kind": "matrix", + "paths": [ + "scripts/run_causal_peft_ftpo.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration; SLM-121 causal PEFT FTPO plan/fixture CLI" + } + ] + }, + "matrix.corruption_curriculum": { + "version": "v1", + "kind": "matrix", + "paths": [ + "scripts/run_corruption_curriculum.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration; SLM-120 near-solved semantic corruption-curriculum plan/fixture CLI" + } + ] + }, + "matrix.grammar": { + "version": "v2", + "kind": "matrix", + "paths": [ + "scripts/run_grammar_matrix.py" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "Persist OpenFeature training/evaluation flag snapshots in grammar-matrix rows" + }, + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration" + } + ] + }, + "matrix.perf": { + "version": "v1", + "kind": "matrix", + "paths": [ + "scripts/run_perf_matrix.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration" + } + ] + }, + "matrix.quality": { + "version": "v7", + "kind": "matrix", + "paths": [ + "scripts/run_quality_matrix.py" + ], + "history": [ + { + "version": "v7", + "date": "2026-07-24", + "note": "Retain authoritative evaluation criteria and runner provenance in quality-matrix summaries" + }, + { + "version": "v6", + "date": "2026-07-23", + "note": "Persist OpenFeature training/evaluation flag snapshots in quality-matrix rows" + }, + { + "version": "v5", + "date": "2026-07-22", + "note": "Centralize design-context conditioning in the shared strict compiler-tree policy inherited by V9 and V10" + }, + { + "version": "v4", + "date": "2026-07-21", + "note": "no-bump: restore historical runtime-symbol-feature experiment descriptions while documenting the separate opaque production projection; matrix dispatch and recipes unchanged" + }, + { + "version": "v4", + "date": "2026-07-21", + "note": "add SLM-210 --matrix-set sde5-floor-escape dispatch for preregistered floor-escape wiring fixture" + }, + { + "version": "v3", + "date": "2026-07-20", + "note": "add SLM-185 default-off --resolution-manifest integration; apply_resolution_manifest annotates summary with semantic_resolution labels without altering raw metrics or gates" + }, + { + "version": "v2", + "date": "2026-07-20", + "note": "no-bump: add SLM-184 default-off fixture comment/example next to evaluate_suites; runner behavior unchanged" + }, + { + "version": "v2", + "date": "2026-07-19", + "note": "add SLM-108 external-ceiling matrix set dispatch (--matrix-set external-ceiling)" + }, + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration" + } + ] + }, + "matrix.slm189_bridge_planner": { + "version": "ffe2-01-v1", + "kind": "matrix", + "paths": [ + "scripts/run_bridge_planner_audit.py", + "docs/design/iter-slm189-bridge-planner-20260721.json", + "docs/design/iter-slm189-bridge-planner-20260721.md" + ], + "history": [ + { + "version": "ffe2-01-v1", + "date": "2026-07-21", + "note": "initial registration; SLM-189 FFE2-01 bridge planner exact/dependency/random arm fixture matrix" + } + ] + }, + "matrix.slm190_exact_flow": { + "version": "ffe2-02-v1", + "kind": "matrix", + "paths": [ + "scripts/run_exact_flow_fixture.py", + "docs/design/iter-slm190-exact-flow-20260721.json", + "docs/design/iter-slm190-exact-flow-20260721.md" + ], + "history": [ + { + "version": "ffe2-02-v1", + "date": "2026-07-21", + "note": "initial registration; SLM-190 FFE2-02 exact CTMC enumeration/path/sample/objective/lumpability fixture matrix" + } + ] + }, + "matrix.slm191_termination_matrix": { + "version": "ffe2-03-v1", + "kind": "matrix", + "paths": [ + "scripts/run_termination_matrix.py", + "tests/test_scripts/test_run_termination_matrix.py", + "docs/design/iter-slm191-termination-matrix-20260721.json", + "docs/design/iter-slm191-termination-matrix-20260721.md" + ], + "history": [ + { + "version": "ffe2-03-v1", + "date": "2026-07-23", + "note": "no-bump: minify the unchanged measured JSON payload to restore Vercel function-size headroom; semantic JSON content and matrix behavior are identical" + }, + { + "version": "ffe2-03-v1", + "date": "2026-07-21", + "note": "no-bump: CLI test passes --no-write-design-docs so committed design docs are not overwritten during pytest; matrix semantics unchanged" + }, + { + "version": "ffe2-03-v1", + "date": "2026-07-21", + "note": "initial registration; SLM-191 FFE2-03 termination-policy fixture matrix" + } + ] + }, + "matrix.slm212_debt_routing": { + "version": "sde5-05-v1", + "kind": "matrix", + "paths": [ + "scripts/run_slm212_debt_routing_fixture.py", + "docs/design/iter-slm212-debt-routing-20260721.json", + "docs/design/iter-slm212-debt-routing-20260721.md" + ], + "history": [ + { + "version": "sde5-05-v1", + "date": "2026-07-21", + "note": "initial registration; SLM-212 SDE5-05 constraint-debt routing MaskGIT/LTR/ASAp matched-budget fixture matrix" + } + ] + }, + "matrix.verified_solver": { + "version": "vss4-02-v1", + "kind": "matrix", + "paths": [ + "src/slm_training/harnesses/experiments/verified_solver_matrix.py", + "src/slm_training/harnesses/experiments/vss4_campaign.py" + ], + "history": [ + { + "version": "vss4-02-v1", + "date": "2026-07-18", + "note": "initial registration; mirrors verified_solver_matrix.MATRIX_VERSION" + } + ] + }, + "model.global_semantic_critic": { + "version": "v1", + "kind": "model", + "paths": [ + "src/slm_training/models/global_semantic_critic.py", + "src/slm_training/models/global_semantic_critic_selector.py", + "tests/test_models/test_global_semantic_critic.py", + "tests/test_models/test_global_semantic_critic_selector.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-150 SPV2-02 global semantic energy/value critic baseline wiring/fixture harness" + } + ] + }, + "model.legal_action_scorer": { + "version": "v1", + "kind": "model", + "paths": [ + "src/slm_training/models/legal_action_scorer.py", + "scripts/run_slm154_legal_action_scorer_fixture.py", + "tests/test_models/test_legal_action_scorer.py", + "docs/design/iter-slm154-legal-action-scorer-20260720.json", + "docs/design/iter-slm154-legal-action-scorer-20260720.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-154 SPV3-01 capacity-matched autoregressive compiler-legal action scorer baseline wiring/fixture harness" + } + ] + }, + "model.legal_edit_batch": { + "version": "v2", + "kind": "model", + "paths": [ + "src/slm_training/models/legal_edit_batch.py", + "tests/test_models/test_legal_edit_batch.py" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-23", + "note": "add label-free runtime packing that preserves exact candidate membership and marks every inference candidate UNKNOWN until independently certified" + }, + { + "version": "v1", + "date": "2026-07-23", + "note": "initial registration; objective-neutral exact dynamic candidate ragged batch, masks, multi-positive targets, padding, and gathered projection seam" + } + ] + }, + "model.quantization": { + "version": "v5", + "kind": "harness", + "paths": [ + "src/slm_training/models/quantization/", + "src/slm_training/models/local_action_head.py" + ], + "history": [ + { + "version": "v5", + "date": "2026-07-20", + "note": "use deterministic action-index mapping in GlobalMaskedHead and ResidualTritPlaneHead to eliminate per-process hash randomization" + }, + { + "version": "v4", + "date": "2026-07-19", + "note": "fix(cap3-05): honest physical-cost ledger rounding and byte-accounting for equal-byte ladder" + }, + { + "version": "v3", + "date": "2026-07-19", + "note": "CAP4-04 compiler-routed block sparsity and state-family experts merged on top of CAP4-02" + }, + { + "version": "v2", + "date": "2026-07-18", + "note": "CAP4-02 adaptive residual-plane scheduling (structural floor + entropy/margin/sensitivity/learned-router/oracle)" + }, + { + "version": "v1", + "date": "2026-07-18", + "note": "initial registration (CAP4-01 residual ternary planes and local scorer head)" + } + ] + }, + "model.recursive_denoiser": { + "version": "v18", + "kind": "harness", + "paths": [ + "src/slm_training/models/recursive_denoiser.py", + "scripts/run_slm138_recursive_denoiser_fixture.py", + "tests/test_models/test_recursive_denoiser.py", + "docs/design/iter-slm138-recursive-denoiser-20260720.json", + "docs/design/iter-slm138-recursive-denoiser-20260720.md", + "src/slm_training/models/rng_contract.py", + "scripts/run_rsc_a02_depth_aux_mode_factorial.py", + "docs/design/iter-slm138-recursive-denoiser-20260721.json", + "docs/design/iter-slm138-recursive-denoiser-20260721.md", + "docs/design/iter-rsc-a03-rng-determinism-contract-20260721.json", + "docs/design/iter-rsc-a03-rng-determinism-contract-20260721.md", + "docs/design/iter-rsc-a04-parity-claim-correction-20260721.json", + "docs/design/iter-rsc-a04-parity-claim-correction-20260721.md", + "src/slm_training/models/recursive_control_arms.py", + "docs/design/iter-rsc-a05-matched-recursive-control-arms-20260721.json", + "docs/design/iter-rsc-a05-matched-recursive-control-arms-20260721.md", + "docs/design/iter-slm138-recursive-denoiser-20260723.json", + "docs/design/iter-slm138-recursive-denoiser-20260723.md", + "docs/design/iter-slm279-depth-supervision-correction-20260723.json", + "docs/design/iter-slm279-depth-supervision-correction-20260723.md", + "docs/design/iter-slm282-recurrence-health-20260723.json", + "docs/design/iter-slm282-recurrence-health-20260723.md" + ], + "history": [ + { + "version": "v18", + "date": "2026-07-23", + "note": "add default-preserving SLM-243 current, delta-only, LayerScale, gated, true-empty-F, and private-norm recurrence controls" + }, + { + "version": "v17", + "date": "2026-07-23", + "note": "expose the exact additive z0 components and default-off evaluation-only recurrence path cuts for SLM-232 without changing the default transition" + }, + { + "version": "v16", + "date": "2026-07-23", + "note": "extract the canonical recurrence into mutation-free initial_transition_state and transition_step APIs so state Jacobians use the exact normal forward math and fixed runtime-symbol conditioning" + }, + { + "version": "v15", + "date": "2026-07-23", + "note": "no-bump: register the clean squash-merged v15 SLM-282 negative recurrence-health JSON and markdown, superseding the withdrawn unmerged v14 positive artifact after raw-example gating exposed the seed-1 regression" + }, + { + "version": "v15", + "date": "2026-07-23", + "note": "no-bump: make the v15 raw-example gate fail inconclusive on missing or duplicate curve depths, example ids, curves, or matched-control records; experiment and disposition semantics are unchanged" + }, + { + "version": "v15", + "date": "2026-07-23", + "note": "SLM-282 preregistration correction: evaluate the exact CE(final) <= CE(previous) <= CE(r=1) condition independently for every raw example and record example ids on adjacent-depth failures, preventing token-weighted aggregation from averaging away a later-depth regression" + }, + { + "version": "v14", + "date": "2026-07-23", + "note": "no-bump: preserve external --output-dir support by using output-dir URIs when AgentV artifacts are outside the repository; this completes the v14 portable-path correction without changing experiment semantics" + }, + { + "version": "v14", + "date": "2026-07-23", + "note": "SLM-282 evidence-writer correction: serialize AgentV spec and artifact paths relative to the repository before mirroring recurrence-health reports into docs/design, while retaining absolute paths for the runtime SDK invocation" + }, + { + "version": "v13", + "date": "2026-07-23", + "note": "SLM-282 (LAR0-02) recurrence-health audit: add the fixed as_is versus fixture-only residual_delta matched-control runner at logical depths 1/2/4 and two disjoint corruption schedules, canonical final-depth-only training with a separate evaluation draw, actual target/noisy/mask digest matching, one-forward anytime curves, seeded detached finite-difference directional gains, exact preregistered disposition, raw failure persistence, and AgentV fixture-not-ship publication" + }, + { + "version": "v12", + "date": "2026-07-23", + "note": "SLM-282 (LAR0-02) diagnostics foundation: add opt-in RecursiveDepthDiagnosticsV1 records with detached per-example y/z states, actual updates, norms, pre-state update ratios, and masked CE/accuracy/entropy/KL curves; add a diagnostics-gated fixture-only residual_delta transition mode while preserving the historical as_is forward path and all production constructor/forward defaults" + }, + { + "version": "v11", + "date": "2026-07-23", + "note": "no-bump: refresh only the clean fixture code_commit and timestamp after replaying the evidence stack onto squash-merged main; all measured arithmetic and objective fields are byte-identical" + }, + { + "version": "v11", + "date": "2026-07-23", + "note": "no-bump: register the clean committed SLM-138 rerun and dedicated SLM-279 correction-only JSON and markdown produced by the v11 fixture contract" + }, + { + "version": "v11", + "date": "2026-07-23", + "note": "SLM-279: make the SLM-138 fixture use the canonical final-primary plus intermediate-only auxiliary objective, emit the complete objective decomposition and recipe, and project dedicated correction-only old-buggy versus corrected arithmetic evidence" + }, + { + "version": "v10", + "date": "2026-07-23", + "note": "no-bump: isolate the SLM-138 fixture checkpoint in a process-unique temporary directory so concurrent changed-test shards cannot read a partially written archive; objective math and emitted evidence are unchanged" + }, + { + "version": "v10", + "date": "2026-07-23", + "note": "no-bump: isolate the SLM-138 fixture checkpoint in a process-unique temporary directory so concurrent changed-test shards cannot read a partially written archive; objective math and emitted evidence are unchanged" + }, + { + "version": "v10", + "date": "2026-07-23", + "note": "no-bump: make SLM-138 fixture and recursive-depth tests explicitly name their existing all-depth semantics while preserving their objective arithmetic" + }, + { + "version": "v10", + "date": "2026-07-23", + "note": "no-bump: extract the behavior-identical normalized recursive-depth loss reduction into a pure helper and make the production objective plus exact arithmetic and gradient-isolation tests share it" + }, + { + "version": "v10", + "date": "2026-07-21", + "note": "no-bump: updated tests/test_models/test_recursive_denoiser.py to expect the SLM-242 construction-time validation raise instead of a training_loss-time raise; the fail-closed invariant and error message content are unchanged, only the call site that triggers the error moved earlier" + }, + { + "version": "v10", + "date": "2026-07-21", + "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v10 revision (d6ff9ceb, code_dirty=false) to embed arm H (control_arm_table row). Content-only refresh -- H's row: 52,288 total parameters (+9,248 vs A, matching recursive_zstate_parameter_delta(d_model=32, max_len=256), identical to B's own delta), block_evaluations_per_forward=4 (same as B). No code change; arm_f_dual_view's active_parameter_count entries shifted slightly (68812->68818, 51922->51928) as an honest side effect of constructing one additional arm (H) earlier in the same run, shifting the ambient global-RNG position for the dual-view towers' random initial weights -- same class of drift every prior arm addition already caused; not a behavior change. Every other arm's row unchanged from the prior v9 no-bump regeneration." + }, + { + "version": "v10", + "date": "2026-07-21", + "note": "SLM-241 (RSC-A05) second follow-up: built control arm H (stop-gradient recurrence) -- the last of the eight arms; SLM-241 now has all 8 arms addressed. SharedRecursiveDenoiserTower gains detach_between_steps: bool = False (src/slm_training/models/recursive_denoiser.py): when True, recursive_outputs runs the identical forward recurrence as detach_between_steps=False (same shared blocks, same y/z update equations, byte-identical forward values for identical seed/weights/inputs) but calls .detach() on the carried-forward y/z at the end of every recursion step except the last, so a later step's loss cannot backpropagate through the recurrent chain into an earlier step's block application (though the shared weights still receive gradient from every step's own application of them). recursive_outputs also gains an additive, opt-in return_step_boundaries: bool = False flag exposing the real pre-detach y/z tensor at each step's exit, for gradient-mechanism tests. recursive_control_arms.py: H moved out of DEFERRED_ARM_IDS into BUILT_ARM_IDS (now empty, all 8 built); ARM_DENOISER_ARCH['H'] reuses B's exact 'shared_recursive' string (no new arch value -- same reuse convention as arm G), construct_arm_tower passes detach_between_steps=(arm_id == 'H'). rng_contract.py: documents (no code change to NAMESPACE_OFFSETS) that H deliberately gets no new arch_specific namespace, since it declares literally the same parameter names/shapes as B -- a fairness report must still never include both B and H together (same restriction already established for B+G), verified by a new test. tests/test_models/test_recursive_denoiser.py: +8 net tests (forward-identity bit-for-bit vs B; a hook-based gradient-divergence mechanism test proving the step-1/step-2 boundary tensor's backward hook fires for B and never fires for H when the loss is the last-depth-only logits; same-step shared-weight gradient still flows in H; params/block-evals match B exactly; config/checkpoint roundtrip; init-fairness exclusion + inclusion cases), 108 passing (was 100 + 9 new, 1 removed). Docs: docs/design/iter-rsc-a05-matched-recursive-control-arms-20260721.{json,md} extended with H's forward-identity and gradient-divergence evidence; SLM-241/RSC-A05 status is now complete (all 8 arms built, none deferred)." + }, + { + "version": "v9", + "date": "2026-07-21", + "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v9 revision (d6cf67c8, code_dirty=false) to embed arm E (control_arm_table row). Content-only refresh -- E's row: 52288 total parameters (+9248 vs A, matching recursive_zstate_parameter_delta(d_model=32, max_len=256)), block_evaluations_per_forward=2 (same as A). No code change; arm_f_dual_view section and every other arm's row unchanged from the prior v8 no-bump regeneration." + }, + { + "version": "v9", + "date": "2026-07-21", + "note": "SLM-241 (RSC-A05) follow-up: built control arm E (stacked + matched state capacity) -- new StackedMatchedStateDenoiserTower class (src/slm_training/models/recursive_denoiser.py): an unshared, non-recursive DenoiserTower subclass (same block-evaluation count as arm A) plus a learned state[max_len,d_model] + state_ctx_proj Linear(d_model,d_model) pair shape-matched to arm B's z_latent/ctx_proj, injected once into the initial hidden state before any transition block runs (never recurrently re-applied). Parameter delta over a same-n_layers arm A equals recursive_zstate_parameter_delta(d_model, max_len) exactly (verified against real constructed towers). recursive_control_arms.py: E moved from DEFERRED_ARM_IDS to BUILT_ARM_IDS (only H remains deferred), construct_arm_tower/build_arm_report/ARM_* registries extended. rng_contract.py: added the arch_specific:stacked_matched_state reserved namespace (offset 110_000), same reserved/not-yet-consumed convention as every other arch_specific:* entry. tests/test_models/test_recursive_denoiser.py: +9 tests (arm E structure/block-eval-count/parameter-formula/gradient-consumption/config-roundtrip/init-fairness, plus one amended table assertion and one deferred-arm-set assertion), net 100 passing (was 94 + 6 new, none removed). Docs: docs/design/iter-rsc-a05-matched-recursive-control-arms-20260721.{json,md} extended with arm E's formula/verification/gradient-consumption-test evidence." + }, + { + "version": "v8", + "date": "2026-07-21", + "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v8 revision (f800cccb, code_dirty=false) to embed arm F (control_arm_table row + arm_f_dual_view section). Content-only refresh -- note: this run also surfaced a pre-existing, code-independent tokenizer-vocab-size drift versus the historical doc (43040 vs 42752 stacked-arm total parameters at this fixture config) reproduced identically on a clean checkout of the prior commit (93ac5062) in an isolated worktree, so it predates and is unrelated to this arm-F change; relative deltas/formulas (e.g. B/G +9248 vs A, matching recursive_zstate_parameter_delta) are unaffected and still verified." + }, + { + "version": "v8", + "date": "2026-07-21", + "note": "SLM-241 (RSC-A05) follow-up: built control arm F (unshared depth-matched tower) -- recursive_control_arms.construct_arm_tower('F', ...) now returns a plain DenoiserTower with recursive_steps * recursive_transition_layers independent blocks (no longer NotImplementedError); added build_arm_f_dual_view (block-evaluation-matched vs parameter-nearest real-measured constructions, honest residual on whichever dimension isn't exact); rng_contract.py gained the arch_specific:stacked_depth_matched reserved namespace (offset 100_000); fixture script gained an arm_f_dual_view report field + markdown section. BUILT_ARM_IDS now (A,B,C,D,F,G); DEFERRED_ARM_IDS now (E,H). No change to A/B/C/D/G construction or reported values." + }, + { + "version": "v7", + "date": "2026-07-21", + "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v7 revision (8f524efc, code_dirty=false) to embed the SLM-241 (RSC-A05) control_arm_table section -- content-only refresh of already-committed evidence (verdict/losses/deep-supervision metrics unchanged), no further code/gate/harness change." + }, + { + "version": "v7", + "date": "2026-07-21", + "note": "SLM-241 (RSC-A05): added z_state_mode ('full'|'y_only'|'parameter_free') to SharedRecursiveDenoiserTower -- 'y_only' (control arm C) removes the distinct z state entirely (both F/G updates fold through y, no z_latent/ctx_proj tensor); 'parameter_free' (arm D) keeps the y/z split but replaces the learned z_latent/ctx_proj with a deterministic pooled-context broadcast, so D's total parameter count matches a stacked baseline with the same recursive_transition_layers/n_layers exactly. Added src/slm_training/models/recursive_control_arms.py -- the A-H control-arm registry (A/B/C/D/G built; E/F/H explicitly deferred, raise NotImplementedError), RecursiveControlArmReportV1 resource accounting (never a parity/winner field), and build_control_arm_table. Extended rng_contract.py with arch_specific: namespaces and RecursiveControlInitializationV1 (fail-closed if common tensors were not actually identically initialized across arms) -- additive to the existing disjoint-namespace contract, not a fork. 19 new tests in tests/test_models/test_recursive_denoiser.py." + }, + { + "version": "v7", + "date": "2026-07-21", + "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v7 revision to embed the new control_arm_table section (SLM-241/RSC-A05) -- content-only refresh of already-committed evidence (verdict/losses/deep-supervision metrics unchanged)." + }, + { + "version": "v6", + "date": "2026-07-21", + "note": "no-bump: regenerated iter-slm138-recursive-denoiser-20260721.{json,md} on the clean committed v6 revision (36dd8d28, code_dirty=false) to embed the new architecture_comparison section/field and the objective-decomposition loss warning -- content-only refresh of already-committed evidence (verdict/losses/deep-supervision metrics unchanged), no further code/gate/harness change." + }, + { + "version": "v6", + "date": "2026-07-21", + "note": "SLM-240 (RSC-A04): retracted recursive_denoiser.py's false 'same parameter count and layer names as DenoiserTower' docstring claim (grepped repo-wide -- only that docstring carried it) and rewrote the module docstring to state precisely what R=1 preserves (public interface / compatible tensor shapes) vs what it never claims (output/parameter equivalence, checkpoint layer-name compatibility for the new z-state keys, any parameter-efficiency claim). Added recursive_zstate_parameter_delta(d_model, max_len) -- the exact formula (z_latent + ctx_proj) reproducing the SLM-138 fixture's 9,248-parameter / +14.23% delta from real constructed towers, never hard-coded -- plus ArchitectureComparisonReportV1/compare_denoiser_architectures (interface_compatible, output_shape_compatible, parameter_count_total/_denoiser, active_parameter_count (grad-based, measured), checkpoint_bytes, common/architecture-specific parameter names+shapes, parameter_count_delta(+formula-match flag), block_evaluations_per_forward (verified against real forward-hook counts), estimated_forward_flops, behaviorally_equivalent_under_declared_degeneracy -- deliberately no single 'parity' field, enforced by __post_init__). Renamed the misnamed test_recursive_steps_one_parity_with_denoiser_tower to test_recursive_r1_preserves_denoiser_interface_and_finite_shapes and added 13 new tests (70 total, all passing) covering the delta formula across (d_model, max_len) pairs, parameter-count independence from recursive_steps, transition-layer name/shape 1:1 mapping, explicit R=1 non-equivalence, checkpoint-migration z-state key coverage, and the comparison report's internal consistency/validation. Wired a real ArchitectureComparisonReportV1 into scripts/run_slm138_recursive_denoiser_fixture.py's report + markdown (interface-compatible/parameter-matched/parameter delta/behavioral parity/claim class fields) and added an objective-decomposition warning (referencing SLM-238's recursive_depth_aux_mode) next to the raw stacked/recursive losses. Annotated docs/design/quality-experiment-matrix.md's V18 section and docs/design/research-lineage.md's SLM-138 row -- SLM-138's wiring-only verdict is unchanged, this is a correction/annotation only. See docs/design/iter-rsc-a04-parity-claim-correction-20260721.{json,md}." + }, + { + "version": "v5", + "date": "2026-07-21", + "note": "no-bump: regenerated iter-rsc-a03-rng-determinism-contract-20260721.{json,md}'s embedded determinism_report from the clean committed revision (4411710e, code_dirty=false throughout) after the v5 harness/doc bump landed -- content-only refresh of already-committed evidence, no further code/gate/harness change." + }, + { + "version": "v5", + "date": "2026-07-21", + "note": "SLM-239 (RSC-A03): added src/slm_training/models/rng_contract.py -- an explicit, disjoint RNG namespace contract (model_initialization/shape_probe_inputs/shape_probe_context/training_corruption/training_batch_order/control_only) plus RngCheckpoint (also covers TwoTowerModel's private self._rng random.Random, a second RNG source _mask_targets reads that a torch-only checkpoint does not capture -- found and fixed during this work). Refactored scripts/run_slm138_recursive_denoiser_fixture.py into the required six-phase execution (construction / deterministic shape probes / deterministic pre-update objective decomposition / one optimizer step / deterministic post-update verification via restored corruption-RNG checkpoint / checkpoint round-trip), added a clean-tree evidence gate (--allow-dirty debug override, non-comparable artifacts never land in docs/design/ without it), and a --mode determinism FixtureDeterminismReportV1 built from real repeated executions + call-order permutations. Added 17 new deterministic regression tests (tests/test_models/test_recursive_denoiser.py, 55 total, all passing) covering the RNG contract, probe-order/extra-probe invariance, restored-checkpoint repeated evaluation (and the torch-only-restore-is-insufficient counter-example), differing-corruption-seed field isolation, fixture exit-state independence from caller entry state, the clean-tree gate, version-stamp match, and checkpoint round-trip digest identity. Measured verdict: bit_exact (real run A/run B/permutation digests match; namespace isolation holds) -- see docs/design/iter-rsc-a03-rng-determinism-contract-20260721.{json,md}." + }, + { + "version": "v4", + "date": "2026-07-21", + "note": "SLM-238 (RSC-A02) follow-up: added test_recursive_objective_contract_v2_validates_sum_identities and a schema-presence assertion in test_generated_decomposition_sums_reproduce_scalar_loss_exactly (41 total). Re-ran scripts/run_rsc_a02_depth_aux_mode_factorial.py --mode fixture --corpus-limit 6 to regenerate the committed docs/design/iter-rsc-a02-depth-aux-mode-factorial-20260721.{json,md} after an earlier --mode plan-only smoke check had accidentally clobbered them with the plan-only skeleton; real fixture-arm numbers now committed and byte-verified against the narrative doc." + }, + { + "version": "v3", + "date": "2026-07-21", + "note": "SLM-238 (RSC-A02): added 14 deterministic regression tests for recursive_depth_aux_mode/recursive_depth_aux_weight and the new objective-decomposition telemetry (tests/test_models/test_recursive_denoiser.py, 40 total, all passing). Added scripts/run_rsc_a02_depth_aux_mode_factorial.py -- a bounded 5-arm (A off / B intermediate_only+uniform / C all_depths+uniform / D intermediate_only+lambda=0.3 / E all_depths+lambda=0.3) deterministic-fixture + bounded real-corpus-smoke comparison; calibration/semantics only, no quality or promotion claim." + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "SLM-237 (RSC-A01): added 17 deterministic regression tests for the corrected weighted deep-supervision objective and the new validate_recursive_depth_supervision fail-closed validator (tests/test_models/test_recursive_denoiser.py). Fixed scripts/run_slm138_recursive_denoiser_fixture.py's own latent instance of failure mode #6 -- it applied recursive_depth_supervision_weights=(0.5, 1.0) to the 'stacked' arm too, which the old code silently ignored and the new fail-closed validator correctly rejects; now only the 'shared_recursive' arm sets the weights." + }, + { + "version": "v2", + "date": "2026-07-21", + "note": "SLM-211: add tie_output_embedding control to SharedRecursiveDenoiserTower" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "no-bump: append run-record rows to design markdown only; harness behavior, thresholds, and report content unchanged" + }, + { + "version": "v1", + "date": "2026-07-20", + "note": "initial registration; SLM-138 shared recursive denoiser tower wiring/fixture harness" + } + ] + }, + "model.twotower": { + "version": "v234", + "kind": "harness", + "paths": [ + "src/slm_training/models/twotower.py", + "src/slm_training/models/semantic_connector.py", + "src/slm_training/models/dynamic_pointer_scorer.py", + "src/slm_training/models/choice_tokenizer.py", + "src/slm_training/models/tokenizer.py", + "src/slm_training/models/grammar.py", + "src/slm_training/models/onnx_inference.py", + "src/slm_training/models/checkpoint_migrate.py", + "src/slm_training/models/template_fill.py", + "src/slm_training/dsl/grammar/fastpath/compiler_draft.py", + "src/slm_training/dsl/grammar/fastpath/token_map.py", + "src/slm_training/data/quality.py", + "src/slm_training/data/contract.py", + "src/slm_training/models/adapters/", + "src/slm_training/harnesses/model_build/config.py", + "src/slm_training/harnesses/model_build/factory.py", + "scripts/train_model.py", + "src/slm_training/models/twotower_numeric_gates.py", + "tests/test_models/test_twotower_numeric_gates.py", + "docs/design/iter-slm242-twotower-numeric-gates-20260721.json", + "docs/design/iter-slm242-twotower-numeric-gates-20260721.md" + ], + "history": [ + { + "version": "v234", + "date": "2026-07-24", + "note": "no-bump: add tree-edit checkpoint format migration without changing TwoTower behavior" + }, + { + "version": "v234", + "date": "2026-07-24", + "note": "Persist exact parameter count and serialized FP32 weight-byte estimate in TwoTower checkpoint metadata" + }, + { + "version": "v233", + "date": "2026-07-24", + "note": "expose existing decode-time model-build levers through the train CLI so periodic evaluation receives the configured values" + }, + { + "version": "v232", + "date": "2026-07-24", + "note": "require budgeted pack-owned completion domains for compiler tree, LTR, and one-hole constrained decoding" + }, + { + "version": "v231", + "date": "2026-07-23", + "note": "persist and validate SLM-243 recurrence update, empty-F, and norm modes while migrating old checkpoints to historical defaults" + }, + { + "version": "v231", + "date": "2026-07-23", + "note": "serve compiler-tree slot contracts through runtime feature tables so prompt-derived OpenUI template markers stay the model output contract" + }, + { + "version": "v230", + "date": "2026-07-23", + "note": "Constrain configured optimizers to the supported AdamW and scratch-only Muon hybrid choices" + }, + { + "version": "v229", + "date": "2026-07-23", + "note": "judge symbolic Harness identity rows from their parsed payload instead of the retired English input delimiter" + }, + { + "version": "v228", + "date": "2026-07-23", + "note": "Expose explicit staged capability, prior-certificate, and distillation preflight inputs through model-build configuration and train CLI" + }, + { + "version": "v227", + "date": "2026-07-23", + "note": "Plumb immutable campaign manifest, result, store, and artifact roots through explicit model-promotion configuration" + }, + { + "version": "v226", + "date": "2026-07-23", + "note": "SLM-279: require new configs with non-empty recursive-depth weights to name intermediate_only, all_depths, or legacy_all_depths explicitly; keep empty configs primary-only and migrate pre-field checkpoint/config loads to legacy_all_depths, including tokenizer and stacked-to-recursive migration paths" + }, + { + "version": "v225", + "date": "2026-07-23", + "note": "no-bump: extract the existing normalized recursive-depth loss arithmetic into a pure reducer without changing objective semantics or telemetry" + }, + { + "version": "v225", + "date": "2026-07-22", + "note": "Constrain primitive array items and separators from their generated schema type while preserving component-array and positional-call boundaries" + }, + { + "version": "v224", + "date": "2026-07-22", + "note": "Keep lexer-native numeric literals inside a schema-checked byte automaton and share one token-to-source conversion across compiler and persistent grammar state" + }, + { + "version": "v223", + "date": "2026-07-22", + "note": "Derive component-array boundaries, nonempty typed children, available-content closure, typed binder compatibility, and acyclic references from schema so constrained decode cannot enter impossible continuations" + }, + { + "version": "v222", + "date": "2026-07-22", + "note": "Derive certified fallback markers from the request contract and use a marker-free fallback when no contract exists" + }, + { + "version": "v221", + "date": "2026-07-22", + "note": "Centralize tokenizer sidecar dispatch and preserve lexer-native tokenizer identity through checkpoint migration" + }, + { + "version": "v220", + "date": "2026-07-22", + "note": "Make declared lexer template markers single-use under slot-contract constrained decode" + }, + { + "version": "v219", + "date": "2026-07-22", + "note": "Bind only direct semantic-role obligations globally so nested repeated-owner markers cannot displace standalone siblings" + }, + { + "version": "v218", + "date": "2026-07-22", + "note": "no-bump: add evaluator-only record offset metadata to ModelBuildConfig without changing model construction or decoding" + }, + { + "version": "v218", + "date": "2026-07-22", + "note": "Share marker namespace ownership between semantic obligations and repeated decode so exhausted structural owners add standalone siblings" + }, + { + "version": "v217", + "date": "2026-07-22", + "note": "Scope repeated-instance ownership to structurally open lexer component calls so parent sibling arrays remain open" + }, + { + "version": "v216", + "date": "2026-07-22", + "note": "Apply repeated-instance marker ownership during restricted and compiler-tree path ranking" + }, + { + "version": "v215", + "date": "2026-07-22", + "note": "Use the shared semantic-plan margin to keep template-marker namespaces owned by their repeated lexer component instance" + }, + { + "version": "v214", + "date": "2026-07-22", + "note": "Carry the semantic-plan margin across lexer array delimiters while sibling component requirements remain" + }, + { + "version": "v213", + "date": "2026-07-22", + "note": "Exclude role metadata from component cardinality and preserve repeated lexer families as siblings unless explicit outer-group topology permits nesting" + }, + { + "version": "v212", + "date": "2026-07-22", + "note": "Treat planned container families as satisfying joint semantic roles reachable through their schema descendants" + }, + { + "version": "v211", + "date": "2026-07-22", + "note": "Apply semantic-plan family scores to direct lexer component candidates inside compiler containers" + }, + { + "version": "v210", + "date": "2026-07-22", + "note": "no-bump: keep compiler test fixtures valid under the stricter central slot-inventory companion contract; model behavior is unchanged" + }, + { + "version": "v210", + "date": "2026-07-22", + "note": "Continue lexer compiler calls into an optional schema property when it owns a still-missing bound semantic role" + }, + { + "version": "v209", + "date": "2026-07-22", + "note": "Apply the existing required-slot margin through one shared compiler restricted/tree adapter" + }, + { + "version": "v208", + "date": "2026-07-22", + "note": "Normalize real lexer component paths as bound families for semantic-plan scoring inside an existing container" + }, + { + "version": "v207", + "date": "2026-07-22", + "note": "Floor an immediate lexer compiler container close after every declared template symbol is consumed" + }, + { + "version": "v206", + "date": "2026-07-22", + "note": "no-bump: consume the canonical slot-contract lever set in the retained runtime guard without changing decode behavior" + }, + { + "version": "v206", + "date": "2026-07-22", + "note": "Treat lexer compiler component paths as bound component candidates in the shared slot-family scorer" + }, + { + "version": "v205", + "date": "2026-07-22", + "note": "Pass declared semantic-role candidates through the shared slot-family scorer on lexer compiler paths" + }, + { + "version": "v204", + "date": "2026-07-22", + "note": "Rank lexer template symbols by the active grammar call and public schema property at every compiler trie edge" + }, + { + "version": "v203", + "date": "2026-07-22", + "note": "Apply semantic-plan margin to keep planned lexer typed-array subtrees nonempty" + }, + { + "version": "v202", + "date": "2026-07-22", + "note": "Count fulfilled semantic-plan families by tokenizer component IDs so bare lexer symbols cannot be forced repeatedly" + }, + { + "version": "v201", + "date": "2026-07-22", + "note": "Keep topology-free lexer semantic-plan obligations on bound families so they cannot override the trained root-role decision" + }, + { + "version": "v200", + "date": "2026-07-22", + "note": "Apply prompt-semantic component obligations during lexer compiler path selection instead of silently ignoring the configured plan" + }, + { + "version": "v199", + "date": "2026-07-22", + "note": "Build prompt-semantic component obligations for the lexer compiler instead of rejecting its supported scoring path" + }, + { + "version": "v198", + "date": "2026-07-22", + "note": "no-bump: record the matched E735 training and smoke evidence for the full-head root-arity objective" + }, + { + "version": "v198", + "date": "2026-07-22", + "note": "Train root-reference arity against the full inference head so high-arity tail logits receive negative supervision" + }, + { + "version": "v197", + "date": "2026-07-22", + "note": "Reject rank-calibrated lexer root arity after it caused three of three bounded smoke decodes to time out" + }, + { + "version": "v196", + "date": "2026-07-22", + "note": "Calibrate lexer root-reference arity weight one as interpolation to the trained continue-or-stop class" + }, + { + "version": "v195", + "date": "2026-07-22", + "note": "Remove unreachable lexer root-reference identity ranking while retaining strict serving checkpoint validation for enabled root heads" + }, + { + "version": "v194", + "date": "2026-07-22", + "note": "Add lexer-native root-reference identity supervision and compiler-path ranking; reject serving checkpoints missing enabled root heads" + }, + { + "version": "v193", + "date": "2026-07-22", + "note": "Fail closed when a learned decode head is enabled without a nonzero owning checkpoint training objective" + }, + { + "version": "v192", + "date": "2026-07-22", + "note": "Add lexer-native root-reference arity supervision and root-list continue-stop scoring" + }, + { + "version": "v191", + "date": "2026-07-21", + "note": "no-bump: adjust ModelBuildConfig policy regression expectations; TwoTower model behavior is unchanged" + }, + { + "version": "v191", + "date": "2026-07-21", + "note": "Apply atomic evaluation policy presets during ModelBuildConfig construction" + }, + { + "version": "v190", + "date": "2026-07-22", + "note": "Fail closed when enabled decode levers cannot execute for the selected output tokenizer and compiler mode" + }, + { + "version": "v189", + "date": "2026-07-22", + "note": "E726: fail closed when choice-only root-reference levers are requested with lexer or compositional output" + }, + { + "version": "v188", + "date": "2026-07-21", + "note": "E716: keep non-string literal markers out of schema-typed STRING compiler candidates" + }, + { + "version": "v187", + "date": "2026-07-21", + "note": "centralize model and training levers while reducing the canonical wall-budget default and maximum to two minutes" + }, + { + "version": "v186", + "date": "2026-07-21", + "note": "Enforce symbol-only output tokenization, remove the free-form LIT_STR channel, and invalidate pre-v2 output-contract checkpoints" + }, + { + "version": "v185", + "date": "2026-07-21", + "note": "E713: compile explicit group-inside-outer-component prompt topology into a verified nested root plan" + }, + { + "version": "v184", + "date": "2026-07-21", + "note": "E712: preserve explicit component counts across arbitrary descriptive modifiers without borrowing counts across phrase boundaries" + }, + { + "version": "v183", + "date": "2026-07-21", + "note": "E711: avoid unprompted joint role carriers when the prompt-planned families can cover the full role group within schema capacity" + }, + { + "version": "v182", + "date": "2026-07-21", + "note": "E709: finalize enum-slot protection after semantic biases and close fully covered terminal Stack lists by margin" + }, + { + "version": "v181", + "date": "2026-07-21", + "note": "add opaque request-local identity to opt-in bound/choice generation while preserving historical name-aware checkpoint semantics and retained v180 carrier behavior; exclude projected marker names from scoring while allowing declared typed authority; add verified exact-choice results, persistent typed scope primitives, and narrow one-hole deterministic MaskGIT completion" + }, + { + "version": "v180", + "date": "2026-07-21", + "note": "E708: retain only a newly created missing-slot carrier in terminal Stack aggregation" + }, + { + "version": "v179", + "date": "2026-07-21", + "note": "E707: restore v177 behavior after terminal-root completeness improved Rico coverage but regressed adversarial fidelity" + }, + { + "version": "v178", + "date": "2026-07-21", + "note": "E707: pair one bounded missing-slot carrier with complete terminal-root reference aggregation" + }, + { + "version": "v177", + "date": "2026-07-21", + "note": "E706: restore v175 behavior after bounded missing-slot carrier routing was neutral across the five-suite replay" + }, + { + "version": "v176", + "date": "2026-07-21", + "note": "E706: bind one still-missing visible slot to a legal direct carrier before verified semantic-plan root closure" + }, + { + "version": "v175", + "date": "2026-07-21", + "note": "E703 rejection: restore v173 repeated-slot scoring after enum-property abstention left all 19 matched predictions unchanged" + }, + { + "version": "v174", + "date": "2026-07-21", + "note": "E703: prevent repeated-instance visible-slot flooring from overriding enum-valued component properties" + }, + { + "version": "v173", + "date": "2026-07-21", + "note": "E702 rejection: restore v171 joint-role planning after counted Callout siblings displaced planned Cards and regressed Held-out and Rico structure" + }, + { + "version": "v172", + "date": "2026-07-21", + "note": "E702: count another preferred joint-role carrier when existing instances cannot cover the accumulated namespace roles" + }, + { + "version": "v171", + "date": "2026-07-21", + "note": "E701 r6: abstain from capacity when roles lack direct property matches and preserve carriers under explicitly typed container descendants" + }, + { + "version": "v170", + "date": "2026-07-21", + "note": "E701 r5: materialize a replacement role carrier only when a directly compatible planned carrier is capacity-exhausted, preserving ordinary nested reachability" + }, + { + "version": "v169", + "date": "2026-07-21", + "note": "E701 r2: materialize enough counted instances for reachable role carriers instead of leaving their bindings advisory" + }, + { + "version": "v168", + "date": "2026-07-21", + "note": "E701: enforce distinct compatible string-property capacity for jointly introduced role carriers and cover the actual broad Form candidate set" + }, + { + "version": "v167", + "date": "2026-07-21", + "note": "E700 rejection: restore v165 role planning after property-capacity matching changed the plan but left all emitted predictions and aggregate quality unchanged" + }, + { + "version": "v166", + "date": "2026-07-21", + "note": "E700: require existing planned component instances to have distinct compatible public string properties before binding another visible role" + }, + { + "version": "v165", + "date": "2026-07-21", + "note": "Merge deterministic active-runtime completion from v94 with the retained E698 likeness abstention and role-planning behavior" + }, + { + "version": "v164", + "date": "2026-07-21", + "note": "Merge the non-finite bound-role mask guard with the retained E698 likeness abstention and enum-role disambiguation behavior" + }, + { + "version": "v163", + "date": "2026-07-21", + "note": "E699 rejection: restore v159 role candidates after form aliases destabilized joint role planning" + }, + { + "version": "v162", + "date": "2026-07-21", + "note": "E699 rejection: restore pre-capacity role planning after the revisit destabilized Form role ownership; retain only explicit form-role alias alignment" + }, + { + "version": "v161", + "date": "2026-07-21", + "note": "E699 correction: align form-role aliases so capacity planning recognizes placeholder/value string properties on public input controls" + }, + { + "version": "v160", + "date": "2026-07-21", + "note": "E699: bound planned-family role reuse by distinct public string-property capacity after likeness abstention removes the false Form owner" + }, + { + "version": "v159", + "date": "2026-07-21", + "note": "E698 r2 correction: disambiguate a visible semantic role when exactly one compatible family names it in a public enum" + }, + { + "version": "v158", + "date": "2026-07-21", + "note": "E698: exclude adjectival component-like mentions from the shared prompt semantic plan" + }, + { + "version": "v157", + "date": "2026-07-21", + "note": "E697 r2 correction: defer ambiguous aggregate families until concrete remaining planned families are emitted" + }, + { + "version": "v156", + "date": "2026-07-21", + "note": "E697: order a planned parent before planned families reachable through its required non-alternative schema paths" + }, + { + "version": "v155", + "date": "2026-07-21", + "note": "E696 rejection: restore v152 behavior after scalar literal fallback left Form placeholder spam unchanged and generated arbitrary operational names" + }, + { + "version": "v154", + "date": "2026-07-21", + "note": "E696 r2 correction: prefer the live grammar's framed string literal before its unavailable fixed empty spelling" + }, + { + "version": "v153", + "date": "2026-07-21", + "note": "E696: route a repeated or incompatible visible slot to the legal empty literal when no unused role-compatible scalar string slot remains" + }, + { + "version": "v152", + "date": "2026-07-21", + "note": "E695 rejection: restore v150 behavior after property-capacity planning only swapped title roles and left placeholder spam, strict meaning, and aggregate quality unchanged" + }, + { + "version": "v151", + "date": "2026-07-21", + "note": "E695: reuse a planned family for another visible role only when distinct public string properties can still bind every assigned role" + }, + { + "version": "v150", + "date": "2026-07-21", + "note": "E694: finalize invalid enum literals with a framed open-vocabulary spelling when the generated row has capacity" + }, + { + "version": "v149", + "date": "2026-07-21", + "note": "E693: plan a public Group component from its unique base noun when no standalone base component exists" + }, + { + "version": "v148", + "date": "2026-07-21", + "note": "E692 r2 correction: treat compact direction tokens as semantically valid enum spellings while repairing invalid fixed literals" + }, + { + "version": "v147", + "date": "2026-07-21", + "note": "E692: normalize invalid fixed enum literals after decode while preserving already-valid enum choices and later decisions" + }, + { + "version": "v146", + "date": "2026-07-21", + "note": "E691: keep transitively slot-bearing array content schema-constrained instead of admitting a raw placeholder" + }, + { + "version": "v145", + "date": "2026-07-21", + "note": "E690: stop schema-reachable wrapper continuation after all of that component's explicit planned role bindings are covered" + }, + { + "version": "v144", + "date": "2026-07-21", + "note": "E689 r2 correction: preserve established schema-role binding at positional slots while testing planned parent descendant disambiguation" + }, + { + "version": "v143", + "date": "2026-07-21", + "note": "E689: disambiguate visible role carriers through cycle-safe schema descendants of already planned parent families; restore E688 wrapper behavior" + }, + { + "version": "v142", + "date": "2026-07-21", + "note": "E688: prevent schema-reachable wrapper continuation from stealing missing roles outside that component's explicit semantic-plan bindings" + }, + { + "version": "v141", + "date": "2026-07-21", + "note": "E686 rejection: restore the original bounded per-call trace budget after model-local rows could not identify independently generated eval records" + }, + { + "version": "v140", + "date": "2026-07-21", + "note": "E686: bound constrained-selection telemetry per batch row so early examples cannot starve later failure traces" + }, + { + "version": "v139", + "date": "2026-07-21", + "note": "E685: plan a non-house-style carrier when it is the sole schema-compatible family for a visible role" + }, + { + "version": "v138", + "date": "2026-07-21", + "note": "E684: derive singular prompt matches for plural schema component families without multiplying singular compound containers" + }, + { + "version": "v137", + "date": "2026-07-21", + "note": "E682: constrain explicitly planned positional role slots by their active public-schema property name" + }, + { + "version": "v136", + "date": "2026-07-21", + "note": "E681: partition larger visible-role namespaces into maximal disjoint groups covered by specific direct-schema carriers" + }, + { + "version": "v135", + "date": "2026-07-21", + "note": "E680: map visible kicker and heading display roles to schema-valid TextContent carriers" + }, + { + "version": "v134", + "date": "2026-07-21", + "note": "E678 provenance correction: localize E676 joint-role planning inside TwoTower so model-only helpers do not perturb the binding-aware evaluator implementation hash; decode behavior is unchanged" + }, + { + "version": "v133", + "date": "2026-07-21", + "note": "E676: plan one schema component for same-namespace roles only when distinct direct string properties jointly cover them" + }, + { + "version": "v132", + "date": "2026-07-21", + "note": "E675: prefer true at a boolean open property only for the actively authored component family" + }, + { + "version": "v131", + "date": "2026-07-21", + "note": "E674: align visible semantic-role candidates with canonical action and dashboard display aliases for Button.label, Callout.description, and TextContent.text" + }, + { + "version": "v130", + "date": "2026-07-21", + "note": "E673: resolve cycle-safe public schema references when testing whether typed-array items can reach visible slots" + }, + { + "version": "v129", + "date": "2026-07-21", + "note": "E672 rejection: restore retained E666 behavior after public-reference item schemas kept slot reachability inactive" + }, + { + "version": "v128", + "date": "2026-07-21", + "note": "E672: let active typed-array schemas start a minimal slot-bearing item even when the enclosing component was model-introduced" + }, + { + "version": "v127", + "date": "2026-07-21", + "note": "E671 rejection: restore retained E666 behavior after nearest nested ownership remained neutral because the real Carousel was not authored in the semantic plan" + }, + { + "version": "v126", + "date": "2026-07-21", + "note": "E671: combine nested item schemas and role-aware targets with nearest-component ownership through nested arrays" + }, + { + "version": "v125", + "date": "2026-07-21", + "note": "E670 rejection: restore retained E666 behavior after the combined nested-schema and role-aware wrapper policy reproduced E669's wrong Form wrapper" + }, + { + "version": "v124", + "date": "2026-07-21", + "note": "E670: combine nested array schema propagation with schema-allowed semantic-role wrapper selection" + }, + { + "version": "v123", + "date": "2026-07-21", + "note": "E669 rejection: restore retained E666 behavior after nested schema propagation chose a semantically wrong Form wrapper and regressed structure, AST F1, and latency" + }, + { + "version": "v122", + "date": "2026-07-21", + "note": "E669: propagate active array item schemas through nested list frames so inner values remain schema-constrained" + }, + { + "version": "v121", + "date": "2026-07-21", + "note": "E668 rejection: restore retained E666 behavior after schema-role typed-array wrappers produced identical predictions and quality metrics" + }, + { + "version": "v120", + "date": "2026-07-21", + "note": "E668: prefer schema-allowed semantic-role component wrappers for missing visible slots at empty typed-array items" + }, + { + "version": "v119", + "date": "2026-07-21", + "note": "E667 rejection: restore retained E666 behavior after nested typed-array ownership produced identical predictions and quality metrics" + }, + { + "version": "v118", + "date": "2026-07-21", + "note": "E667: apply schema-derived typed-array item margins through nested arrays owned by an authored plan component" + }, + { + "version": "v117", + "date": "2026-07-21", + "note": "E666: normalize completed dynamic literals at schema-enum properties after choice decoding so later model decisions remain unchanged" + }, + { + "version": "v116", + "date": "2026-07-21", + "note": "E659 rejection: restore retained E653 behavior after hard property-role masking collapsed meaningfulness, fidelity, structure, reward, and latency" + }, + { + "version": "v115", + "date": "2026-07-21", + "note": "E659: final-mask visible slots that violate the active positional property's schema role" + }, + { + "version": "v114", + "date": "2026-07-21", + "note": "E658 rejection: restore retained E653 behavior after property-aware margins dropped a required metric and regressed fidelity, structure, and reward" + }, + { + "version": "v113", + "date": "2026-07-21", + "note": "E658: gate raw visible-slot margins by the active positional property's schema role" + }, + { + "version": "v112", + "date": "2026-07-21", + "note": "E657 rejection: restore retained E653 behavior after combined ownership constraints regressed structure, AST F1, and latency without improving strict meaning" + }, + { + "version": "v111", + "date": "2026-07-21", + "note": "E657: combine direct-owner raw-slot constraints across coverage and repeated-plan margins" + }, + { + "version": "v110", + "date": "2026-07-21", + "note": "E656 rejection: restore retained E653 behavior after repeated-slot role ownership alone left predictions and quality unchanged" + }, + { + "version": "v109", + "date": "2026-07-21", + "note": "E656: restrict repeated-plan raw-slot margins to roles directly owned by the active component" + }, + { + "version": "v108", + "date": "2026-07-21", + "note": "E655 rejection: restore retained E653 behavior after direct-owner slot forcing left the matched predictions and quality metrics unchanged" + }, + { + "version": "v107", + "date": "2026-07-21", + "note": "E655: require direct semantic-role ownership before forcing a raw visible slot; transitive owners descend through compatible children" + }, + { + "version": "v106", + "date": "2026-07-21", + "note": "E654 rejection: restore retained E653 v104 after enum flooring reduced fidelity, validity, structure, and reward" + }, + { + "version": "v105", + "date": "2026-07-21", + "note": "E654: retry schema-valid enum flooring after E653 stabilizes nested visible-role ownership" + }, + { + "version": "v104", + "date": "2026-07-21", + "note": "E653: keep visible-role leaf obligations nested when an already-planned component can reach the role through its public schema" + }, + { + "version": "v103", + "date": "2026-07-21", + "note": "Restore v98 behavior after E652 detached metric leaves from their planned Cards and reduced meaningful output." + }, + { + "version": "v102", + "date": "2026-07-21", + "note": "Recognize display value roles as text-compatible so planned containers receive simple visible leaf owners." + }, + { + "version": "v101", + "date": "2026-07-21", + "note": "Restore v96 behavior after E651 enum flooring reduced meaningful output and produced an empty planned Card." + }, + { + "version": "v100", + "date": "2026-07-21", + "note": "Floor schema-valid enum literals or optional closure above invalid dynamic values during constrained decode." + }, + { + "version": "v99", + "date": "2026-07-21", + "note": "Bind compatible visible roles to already-planned families before inferring any additional family instances." + }, + { + "version": "v98", + "date": "2026-07-21", + "note": "Restore v93 behavior after E649 refresh-role binding reduced meaningful output quality despite complete placeholder coverage." + }, + { + "version": "v97", + "date": "2026-07-21", + "note": "Recognize refresh as an action-label semantic role so prompt-planned controls can bind their visible slot." + }, + { + "version": "v96", + "date": "2026-07-20", + "note": "E648: verify semantic-root closures through the dynamic-literal-aware choice tokenizer before official parser validation" + }, + { + "version": "v95", + "date": "2026-07-20", + "note": "E647 r2: deduplicate semantic-root abstention traces by stable error signature rather than changing planned-section counts" + }, + { + "version": "v94", + "date": "2026-07-20", + "note": "E647: emit bounded behavior-neutral semantic-root verifier abstention evidence with error type, message, and planned closure counts" + }, + { + "version": "v93", + "date": "2026-07-20", + "note": "E646 rejection: restore v88 plan-counted root references after all-section reachability closure was prediction- and quality-neutral" + }, + { + "version": "v92", + "date": "2026-07-20", + "note": "E646: close verified semantic roots over every completed top-level element after required plan coverage so unplanned valid sections cannot orphan the root" + }, + { + "version": "v91", + "date": "2026-07-20", + "note": "E644: pair visible-role-inferred families with concrete unused slots and margin-floor only the assigned role at compatible schema positions" + }, + { + "version": "v90", + "date": "2026-07-20", + "note": "E643 rejection: restore v85 prompt-only semantic plans after direct required-string role binding reproduced E641 strict, structure, and latency regressions" + }, + { + "version": "v89", + "date": "2026-07-20", + "note": "E643: bind visible-role-inferred families to compatible required plain-string content arguments while preserving pre-content operational literals" + }, + { + "version": "v88", + "date": "2026-07-20", + "note": "E642 rejection: restore v83 prompt-only semantic plans after root-only inferred families regressed meaning, structure, reward, AST F1, and tail latency" + }, + { + "version": "v87", + "date": "2026-07-20", + "note": "E642: constrain visible-role-inferred semantic-plan families to completed root-section boundaries to prevent nested family spam" + }, + { + "version": "v86", + "date": "2026-07-20", + "note": "E641 rejection: restore v84 prompt-only semantic plans after visible-role completion cut strict v2, structure, and latency despite fidelity and reward gains" + }, + { + "version": "v85", + "date": "2026-07-20", + "note": "E641: complete prompt-planned families with house-style-preferred schema-compatible leaves for visible roles no planned family can own" + }, + { + "version": "v84", + "date": "2026-07-20", + "note": "E640 rejection: restore v82 state and root references after supplemental slot-bearing reference tracking left every policy-matched output and metric unchanged" + }, + { + "version": "v83", + "date": "2026-07-20", + "note": "E640: track visible slots per completed root section and include supplemental slot-bearing sections in verifier-checked semantic root references" + }, + { + "version": "v82", + "date": "2026-07-20", + "note": "E639 rejection: restore v80 root-boundary behavior after sibling coverage sampling changed traces but the verified root omitted those unplanned sections and every policy-matched metric stayed flat" + }, + { + "version": "v81", + "date": "2026-07-20", + "note": "E639: continue from completed root sections with the best legal role-compatible component while visible slots remain" + }, + { + "version": "v94", + "date": "2026-07-21", + "note": "make exact LTR, repair, choice, and complete-compiler singleton decisions bypass neural rows; add honest verified template-plus-bindings generation boundary" + }, + { + "version": "v93", + "date": "2026-07-21", + "note": "Skip role-obligation margin flooring for candidates already blocked to a non-finite score by semantic candidate masks" + }, + { + "version": "v92", + "date": "2026-07-21", + "note": "no-bump: renumber E644 role-obligation evidence to E650 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" + }, + { + "version": "v92", + "date": "2026-07-20", + "note": "E650: pair visible-role-inferred families with concrete unused slots and margin-floor only the assigned role at compatible schema positions; renumbered from the originally launched E644 arm after rebase" + }, + { + "version": "v91", + "date": "2026-07-21", + "note": "no-bump: renumber E643 bound-role-plan evidence to E649 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" + }, + { + "version": "v91", + "date": "2026-07-20", + "note": "E649 rejection: restore v89 prompt-only semantic plans after direct required-string role binding reproduced E647 strict, structure, and latency regressions; renumbered from the originally launched E643 arm after rebase" + }, + { + "version": "v90", + "date": "2026-07-20", + "note": "E649: bind visible-role-inferred families to compatible required plain-string content arguments while preserving pre-content operational literals; renumbered from the originally launched E643 arm after rebase" + }, + { + "version": "v89", + "date": "2026-07-21", + "note": "no-bump: renumber E642 root-only-role-plan evidence to E648 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" + }, + { + "version": "v89", + "date": "2026-07-20", + "note": "E648 rejection: restore v87 prompt-only semantic plans after root-only inferred families regressed meaning, structure, reward, AST F1, and tail latency; renumbered from the originally launched E642 arm after rebase" + }, + { + "version": "v88", + "date": "2026-07-20", + "note": "E648: constrain visible-role-inferred semantic-plan families to completed root-section boundaries to prevent nested family spam; renumbered from the originally launched E642 arm after rebase" + }, + { + "version": "v87", + "date": "2026-07-21", + "note": "no-bump: renumber E641 role-plan-completion evidence to E647 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" + }, + { + "version": "v87", + "date": "2026-07-20", + "note": "E647 rejection: restore v85 prompt-only semantic plans after visible-role completion cut strict v2, structure, and latency despite fidelity and reward gains; renumbered from the originally launched E641 arm after rebase" + }, + { + "version": "v86", + "date": "2026-07-20", + "note": "E647: complete prompt-planned families with house-style-preferred schema-compatible leaves for visible roles no planned family can own; renumbered from the originally launched E641 arm after rebase" + }, + { + "version": "v85", + "date": "2026-07-21", + "note": "no-bump: reconcile E646 evidence to treatment v84 and restoration v85 after integrating SLM-242 and E645 through v83; no model behavior change" + }, + { + "version": "v85", + "date": "2026-07-20", + "note": "E646 rejection: restore v83 state and root references after supplemental slot-bearing reference tracking left every policy-matched output and metric unchanged; renumbered from the originally launched E640 arm after rebase" + }, + { + "version": "v84", + "date": "2026-07-21", + "note": "no-bump: renumber E640 root-slot-reference evidence to E646 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" + }, + { + "version": "v84", + "date": "2026-07-20", + "note": "E646: track visible slots per completed root section and include supplemental slot-bearing sections in verifier-checked semantic root references; renumbered from the originally launched E640 arm after rebase" + }, + { + "version": "v83", + "date": "2026-07-21", + "note": "no-bump: renumber E639 sibling-coverage evidence to E645 after rebase incorporated the independently landed E639-E644 sequence; no model behavior change" + }, + { + "version": "v83", + "date": "2026-07-20", + "note": "E645 rejection: restore v81 numeric-gate and root-boundary behavior after sibling coverage sampling changed traces but the verified root omitted those unplanned sections and every policy-matched metric stayed flat" + }, + { + "version": "v82", + "date": "2026-07-20", + "note": "E645: continue from completed root sections with the best legal role-compatible component while visible slots remain; renumbered from the originally launched E639 arm after rebase" + }, + { + "version": "v81", + "date": "2026-07-21", + "note": "no-bump: validator refinements after broader test run -- allow steps=0, decode_min_content=-1, finite (not non-negative) slot_component_lexeme/span priors, and a human-readable max_wall_minutes message; no change to the fail-closed intent or to successfully validated configs" + }, + { + "version": "v81", + "date": "2026-07-21", + "note": "SLM-242 (RSC-A06): fail-closed numeric/schedule validation gate for every TwoTower weight and schedule vector; adds src/slm_training/models/twotower_numeric_gates.py, ModelBuildConfig.__post_init__ numeric gate, TwoTowerConfig.__post_init__ numeric gate, CLI --recursive-depth-aux-mode/--recursive-depth-aux-weight in scripts/train_model.py" + }, + { + "version": "v80", + "date": "2026-07-21", + "note": "no-bump: integrate PR #625 version history for model.twotower; prior PR entries below record the behavior changes" + }, + { + "version": "v69", + "date": "2026-07-21", + "note": "SLM-241 (RSC-A05) second follow-up: added recursive_detach_between_steps: bool = False to TwoTowerConfig (and ModelBuildConfig / _twotower_config_from_build / scripts/train_model.py --recursive-detach-between-steps) -- an orthogonal flag to denoiser_arch/z_state_mode, threaded into SharedRecursiveDenoiserTower(detach_between_steps=...) in the shared_recursive construction branch. Reuses denoiser_arch='shared_recursive' (arm B's/G's arch string; no new value); when True, this is control arm H (stop-gradient recurrence). No change to any other denoiser_arch's behavior; existing shared_recursive checkpoint z_latent/ctx_proj warm-start allowance already covers H (same parameter names as B)." + }, + { + "version": "v68", + "date": "2026-07-21", + "note": "SLM-241 (RSC-A05) follow-up: added stacked_matched_state as a canonical denoiser_arch value (control arm E, stacked + matched state capacity) -- routes to the new StackedMatchedStateDenoiserTower (src/slm_training/models/recursive_denoiser.py): an unshared, non-recursive DenoiserTower variant (denoiser_layers blocks, each called once -- same block-evaluation count as arm A) plus a learned state/state_ctx_proj pair shape-matched to arm B's z_latent/ctx_proj, injected once before the transition blocks run (never recurrently re-applied). Checkpoint loader gains a matching warm-start allowance for denoiser.state/denoiser.state_ctx_proj.* missing keys, mirroring the existing shared_recursive allowance. No change to existing 'stacked'/'stacked_depth_matched'/'shared_recursive'/'shared_recursive_y_only'/'shared_recursive_no_extra_capacity' behavior. CLI --denoiser-arch (scripts/train_model.py) gains the new choice." + }, + { + "version": "v67", + "date": "2026-07-21", + "note": "SLM-241 (RSC-A05) follow-up: added stacked_depth_matched as a canonical denoiser_arch value (control arm F, unshared depth-matched tower) -- routes to a plain DenoiserTower built with recursive_steps * recursive_transition_layers independent transition blocks instead of denoiser_layers, so its block-evaluation count matches the shared-recursive arm's (arm B) exactly. No new tower class; no z-state; no change to existing 'stacked'/'shared_recursive'/'shared_recursive_y_only'/'shared_recursive_no_extra_capacity' behavior. CLI --denoiser-arch (scripts/train_model.py) gains the new choice." + }, + { + "version": "v66", + "date": "2026-07-21", + "note": "SLM-241 (RSC-A05): denoiser construction now fails closed on an unrecognized denoiser_arch (KNOWN_DENOISER_ARCHES) instead of silently falling back to 'stacked' -- added shared_recursive_y_only and shared_recursive_no_extra_capacity as canonical denoiser_arch values (control arms C/D), routed through SharedRecursiveDenoiserTower's new z_state_mode via SHARED_RECURSIVE_ARCH_Z_STATE_MODES. CLI --denoiser-arch (scripts/train_model.py) gains the two new choices. No change to existing 'stacked'/'shared_recursive' behavior." + }, + { + "version": "v65", + "date": "2026-07-21", + "note": "SLM-238 (RSC-A02) follow-up: added the required RecursiveObjectiveContractV2 schema (src/slm_training/models/twotower.py) -- a typed, validated view over the objective-decomposition telemetry (mode, aux_weight, primary/intermediate/final/combined loss terms) whose __post_init__ re-checks both required sum identities at construction time. training_loss now also populates last_training_metrics['recursive_objective_contract'] (a dict) alongside the existing flat scalar fields, which remain the single source of truth. Additive only -- no existing metric value, gate, or default changed." + }, + { + "version": "v64", + "date": "2026-07-21", + "note": "SLM-238 (RSC-A02): added recursive_depth_aux_mode (off | intermediate_only | all_depths | legacy_all_depths) and recursive_depth_aux_weight to TwoTowerConfig, extending validate_recursive_depth_supervision/ValidatedDepthSupervision (not forking it) to accept mode/expected-depth-range. Makes the final recursion depth's double-counting (rec_out['logits'] == rec_out['depth_logits'][-1], so the primary term already reflects it once) an explicit, named choice: 'intermediate_only' structurally never indexes depth_logits[-1]; 'all_depths' counts it exactly once more in the auxiliary term. Added exact objective-decomposition telemetry (primary_final_reconstruction_loss, recursive_intermediate_aux_loss, recursive_final_depth_aux_contribution, recursive_depth_aux_weight, combined_training_loss), always populated (explicit 0.0 when disabled, never omitted). resolve_recursive_depth_aux_mode/migrate_recursive_depth_aux_config give any pre-existing config or checkpoint (recursive_depth_aux_mode absent) byte-identical 'legacy_all_depths' behavior -- 'off' is the new default only when recursive_depth_supervision_weights was never set. No production default changed; no adoption decision (deferred to SLM-233)." + }, + { + "version": "v63", + "date": "2026-07-21", + "note": "SLM-237 (RSC-A01): fixed the recursive_depth_supervision_weights training_loss defect -- the loop bound each per-depth weight but never multiplied by it, computing sum(L_d)/sum(w_d) (an unweighted mean) instead of the intended sum(w_d*L_d)/sum(w_d). Added validate_recursive_depth_supervision/ValidatedDepthSupervision: fails closed on unsupported architecture, length mismatch, negative/NaN/inf weights, and all-zero configs (previously all silently accepted/truncated/ignored); empty tuple remains feature-off with no behavior change. Added recursive_depth_weight_{d}/recursive_depth_weighted_contribution_{d}/recursive_depth_supervision_weight_sum/recursive_depth_supervision_enabled telemetry. RSC-A02 (whether the final depth belongs in the auxiliary term) is out of scope." + }, + { + "version": "v62", + "date": "2026-07-20", + "note": "E630: _required_slot_margin_bias now gates on the active argument position's schema, mirroring _schema_role_slot_bias's own accepts_slot check (x-openui-placeholder-flagged on a component frame, or schema-can-reach-a-visible-slot on an object frame) -- fixes E629's traced rico_eval_test_25 frame_depth>=1 over-stuffing regression (one Button absorbing 5 missing slots across all 5 positional arguments, since the bias's old_max+margin floor always wins against schema_value_bias/schema_opaque_bias/schema_enum_close_bias/schema_opaque_close_bias regardless of margin size); default weight (0.0) and E617 contract-gating unchanged. Also found: this same fix reverts E626/E628's previously-reported ood_dashboard_01 margin=2 gain back to control (byte-identical), since that gain was mechanistically the identical over-stuffing behavior and partly an artifact of _is_meaningful_program's literal \"Card([])\" substring check missing an empty-children Card padded with non-content placeholder args; a widened n=19 resweep with the fix applied also surfaces a third, previously invisible regression (smoke_hero_01) -- net binary-gate effect at margin>=1 is now below control (12/19 vs 13/19), not flat. See docs/design iter-e630." + }, + { + "version": "v61", + "date": "2026-07-20", + "note": "E628: _required_slot_margin_bias now excludes frame_depth == 0 (no component/object frame open yet) from its target set, fixing E627's traced margin=6 root hijack -- the bias only fires once decoding is already inside a real component/object frame; default weight (0.0) and E617 contract-gating unchanged" + }, + { + "version": "v60", + "date": "2026-07-20", + "note": "no-bump: E627 adds required_slot_margin_applications/required_slot_margin_choice_changes DecodeStats counters and a bounded required_slot_margin constrained_selection_traces entry (row/position/frame_depth/candidate kinds) purely to observe required_slot_margin_decode_weight's (E626) existing behavior; no bias formula, weight, or default changed" + }, + { + "version": "v60", + "date": "2026-07-20", + "note": "E626: add required_slot_margin_decode_weight, a default-off decode-time margin that floors the best legal candidate filling a still-missing required slot (analogous to semantic_plan_margin_decode_weight for plan families); contract-gated by the E617 slot_contract_constrained_decode guard" + }, + { + "version": "v80", + "date": "2026-07-20", + "note": "E638 rejection: restore v78 behavior after requiring complete slot coverage collapsed Gallery and regressed fidelity, validity, structure, reward, and latency" + }, + { + "version": "v79", + "date": "2026-07-20", + "note": "E638: require complete visible slot-contract coverage before semantic-plan root closure can activate" + }, + { + "version": "v78", + "date": "2026-07-21", + "note": "SLM-212: add default-off constraint_debt_routing_* fields to ModelBuildConfig" + }, + { + "version": "v77", + "date": "2026-07-21", + "note": "SLM-222: add optimizer_name, muon_lr, adamw_lr, weight_decay, muon_momentum, muon_nesterov, and muon_ns_steps fields to ModelBuildConfig" + }, + { + "version": "v76", + "date": "2026-07-21", + "note": "SLM-211: add tie_output_embedding control, deduplicate tied parameters in trainable_parameters/optimizer_parameter_groups, and explicit checkpoint tie migration" + }, + { + "version": "v75", + "date": "2026-07-20", + "note": "E637: count semantic-plan component families from the emitted choice prefix at every nesting depth so legal wrapper descendants satisfy family coverage without duplicate top-level emission" + }, + { + "version": "v74", + "date": "2026-07-20", + "note": "E636: enforce exact component variants in typed arrays, follow public-schema wrappers to missing visible roles, and close legal optional tails after complete slot coverage" + }, + { + "version": "v73", + "date": "2026-07-20", + "note": "E635: combine final pre-content literal routing with property-compatible slot coverage so direct visible slots cannot be floored into non-placeholder component arguments" + }, + { + "version": "v72", + "date": "2026-07-20", + "note": "E634 rejection: restore v70 early pre-content routing after final-boundary enforcement collapsed Auth to TextContent and regressed every aggregate quality metric" + }, + { + "version": "v71", + "date": "2026-07-20", + "note": "E634: split pre-content literal routing from optional opaque penalties and apply it after repeated-slot margins at the final active-property decision boundary" + }, + { + "version": "v70", + "date": "2026-07-20", + "note": "E633 r2 rejection: restore v68 repeated-slot behavior after property-level restriction collapsed Auth to TextContent and regressed every aggregate quality metric" + }, + { + "version": "v69", + "date": "2026-07-20", + "note": "E633 r2: apply repeated-instance distinct-slot margin only at direct placeholder-annotated component properties so it cannot override pre-content operational string routing" + }, + { + "version": "v68", + "date": "2026-07-20", + "note": "E633: when a required operational string is immediately followed by a visible-content property, floor the legal empty literal above visible slots at only that active property boundary" + }, + { + "version": "v67", + "date": "2026-07-20", + "note": "E632 rejection: restore v65 optional-empty-only schema opaque behavior after required non-content string penalties removed Input inventory and regressed every continuous aggregate metric" + }, + { + "version": "v66", + "date": "2026-07-20", + "note": "E632: extend schema_opaque_decode_weight to penalize visible slots in required or optional non-content string arguments while preserving content-annotated properties" + }, + { + "version": "v65", + "date": "2026-07-20", + "note": "E631: when a component owner cannot own any missing visible role, prefer closing it instead of nesting a different role-compatible component at the wrong structural depth" + }, + { + "version": "v64", + "date": "2026-07-20", + "note": "E630 rejection: restore v62 broad schema-role closure behavior after prompt-owned filtering nested Input inside Button and regressed every aggregate quality metric" + }, + { + "version": "v63", + "date": "2026-07-20", + "note": "E630: prefer explicit prompt-owned component candidates over broad schema-only role matches when preventing premature slot-coverage closure" + }, + { + "version": "v62", + "date": "2026-07-20", + "note": "E622: emit bounded slot-coverage closure score traces plus application and immediate-choice-change counters without changing decode scores" + }, + { + "version": "v61", + "date": "2026-07-20", + "note": "E621: require public-schema owner compatibility before a direct visible-slot continuation can outrank frame closure" + }, + { + "version": "v60", + "date": "2026-07-20", + "note": "E621: extend slot_coverage_close_decode_weight to floor a compiler-legal role-compatible component or object-property continuation above premature frame closure while abstaining when compatibility cannot be proven" + }, + { + "version": "v59", + "date": "2026-07-20", + "note": "E617: raise ValueError in _generate_batch_once when schema_role_slot_decode_weight / slot_coverage_close_decode_weight / semantic_plan_typed_array_nonempty_margin_decode_weight / semantic_plan_typed_array_item_margin_decode_weight / semantic_plan_repeated_slot_margin_decode_weight is set without slot_contract_constrained_decode or template_fill_decode, since self._slot_contracts silently stays None otherwise and these biases no-op every step" + }, + { + "version": "v58", + "date": "2026-07-20", + "note": "score visible slots against active inline object properties derived from the public schema" + }, + { + "version": "v57", + "date": "2026-07-20", + "note": "preserve closed object schemas through choice decoding and require declared properties" + }, + { + "version": "v56", + "date": "2026-07-20", + "note": "add SLM-176 P14 action-shortlist mode/config fields, CLI args, and optional retrieve-then-rerank in _select_compiler_path; defaults preserve off behavior" + }, + { + "version": "v55", + "date": "2026-07-20", + "note": "optionally floor the schema-derived item start inside authored typed arrays" + }, + { + "version": "v54", + "date": "2026-07-20", + "note": "optionally start empty slot-bearing typed arrays inside authored prompt-plan components" + }, + { + "version": "v53", + "date": "2026-07-20", + "note": "add SLM-174 SDE2-07 action-alias mode/manifest/name-mode config fields, CLI args, and teacher-init wiring; defaults preserve canonical behavior" + }, + { + "version": "v52", + "date": "2026-07-20", + "note": "optionally floor the best unused visible slot inside each repeated prompt-plan instance" + }, + { + "version": "v51", + "date": "2026-07-20", + "note": "select the deepest repeated semantic-plan ancestor for nested array closure" + }, + { + "version": "v50", + "date": "2026-07-20", + "note": "add default-off repeated-plan-family nested-array close margin and track variadic item counts" + }, + { + "version": "v49", + "date": "2026-07-20", + "note": "add SDE2-03 exposure-targeted rare-action sampling config fields and CLI args; defaults preserve existing behavior" + }, + { + "version": "v48", + "date": "2026-07-20", + "note": "optionally floor verifier-approved semantic-plan root and completion tokens above the best legal score" + }, + { + "version": "v47", + "date": "2026-07-20", + "note": "record bounded score decomposition for verifier-approved semantic-plan root tokens atop default-off SLM-168 pointer plumbing" + }, + { + "version": "v46", + "date": "2026-07-20", + "note": "add SLM-168 pointer_mode / pointer_candidate_source / pointer_* plumbing and standalone dynamic_pointer_scorer module; defaults preserve existing behavior" + }, + { + "version": "v45", + "date": "2026-07-20", + "note": "optionally floor still-required semantic-plan families above the best legal component score" + }, + { + "version": "v44", + "date": "2026-07-20", + "note": "retain all positively biased planned-family candidates outside the diagnostic top-score cutoff" + }, + { + "version": "v43", + "date": "2026-07-20", + "note": "record bounded post-first semantic-plan family score decompositions without changing decode scores" + }, + { + "version": "v42", + "date": "2026-07-20", + "note": "add SLM-166 semantic_connector / train_scope / connector_* plumbing and standalone connector module; defaults preserve existing behavior" + }, + { + "version": "v41", + "date": "2026-07-20", + "note": "extend first-component plan traces through the actual final legal-token selection" + }, + { + "version": "v40", + "date": "2026-07-20", + "note": "record bounded first-component semantic-plan score decompositions in decode evidence" + }, + { + "version": "v39", + "date": "2026-07-20", + "note": "gate first-component prompt-plan scoring on compiler root candidate kinds rather than parser frame emptiness" + }, + { + "version": "v38", + "date": "2026-07-20", + "note": "add a default-off prompt-plan score restricted to the first component-family choice" + }, + { + "version": "v37", + "date": "2026-07-20", + "note": "add SLM-164 legal_margin_mode / targeted_margin_* plumbing; defaults preserve existing behavior" + }, + { + "version": "v36", + "date": "2026-07-20", + "note": "preserve explicit component counts when one descriptive modifier separates the count from the component family" + }, + { + "version": "v35", + "date": "2026-07-20", + "note": "optionally close typed component arrays after all visible slots are covered" + }, + { + "version": "v34", + "date": "2026-07-20", + "note": "add SLM-163 action_embedding_init and action_embedding_train plumbing; defaults preserve existing behavior" + }, + { + "version": "v33", + "date": "2026-07-20", + "note": "no-bump: complete runtime override plumbing for the already registered v33 lever" + }, + { + "version": "v33", + "date": "2026-07-20", + "note": "optionally derive component choices from visible slot roles and the public schema" + }, + { + "version": "v33", + "date": "2026-07-20", + "note": "add SLM-163 action_embedding_init and action_embedding_train plumbing; defaults preserve existing behavior" + }, + { + "version": "v32", + "date": "2026-07-20", + "note": "map visible body and action role aliases to compatible schema properties for prompt-role binding" + }, + { + "version": "v31", + "date": "2026-07-20", + "note": "infer Button in predicted partial plans from authored action and confirmation semantics" + }, + { + "version": "v30", + "date": "2026-07-20", + "note": "score still-missing prompt component families at inline component positions using valid-prefix counts" + }, + { + "version": "v29", + "date": "2026-07-20", + "note": "add default-off legal closure scoring for optional enum-valued component arguments" + }, + { + "version": "v28", + "date": "2026-07-20", + "note": "preserve component-array item schemas in choice decode state so invalid direct placeholder children are compiler-illegal" + }, + { + "version": "v27", + "date": "2026-07-20", + "note": "add default-off visible slot scoring by active content-property owner compatibility" + }, + { + "version": "v26", + "date": "2026-07-20", + "note": "add a default-off legality-preserving score for closing optional unconstrained component arguments" + }, + { + "version": "v25", + "date": "2026-07-20", + "note": "add a default-off legality-preserving bias against visible placeholders in optional unconstrained component arguments" + }, + { + "version": "v24", + "date": "2026-07-20", + "note": "add a default-off legality-preserving schema-value bias that discourages visible placeholders in enum-valued component arguments" + }, + { + "version": "v23", + "date": "2026-07-20", + "note": "bind visible-role gating confidence to complete coverage of the original slot contract so uncovered consumed roles cannot enable it late" + }, + { + "version": "v22", + "date": "2026-07-20", + "note": "abstain from visible-role gating unless every remaining slot has at least one honest role-family candidate" + }, + { + "version": "v21", + "date": "2026-07-20", + "note": "gate the auxiliary learned slot-head bonus to visible role-matching families when honest role evidence is available" + }, + { + "version": "v20", + "date": "2026-07-20", + "note": "derive honest visible slot-to-component candidates from local authored role-family phrases in addition to schema property names" + }, + { + "version": "v19", + "date": "2026-07-20", + "note": "soft-close repeated predicted component instances after their first visible slot so later instances can bind distinct prompt slots" + }, + { + "version": "v18", + "date": "2026-07-20", + "note": "soft-score only still-missing predicted component-family instances so repeated authored requirements can be generated before root closure" + }, + { + "version": "v17", + "date": "2026-07-20", + "note": "preserve repeated authored component mentions in predicted plans and require their family cardinality before verified root closure" + }, + { + "version": "v16", + "date": "2026-07-20", + "note": "soft-follow a compiler- and verifier-valid Stack closure after predicted SemanticPlanV1 component coverage, failing closed without a valid completion" + }, + { + "version": "v15", + "date": "2026-07-20", + "note": "build and terminate a legal Stack root only after predicted SemanticPlanV1 component-family coverage and extend plan binding into its nested child list" + }, + { + "version": "v14", + "date": "2026-07-20", + "note": "compose predicted semantic-plan binding evidence after learned root-reference identity ranking so identity cannot erase the plan factor" + }, + { + "version": "v13", + "date": "2026-07-20", + "note": "soft-rank legal unused terminal-root references whose generated component families match the predicted prompt SemanticPlanV1" + }, + { + "version": "v12", + "date": "2026-07-20", + "note": "consume prompt-derived predicted SemanticPlanV1 component features as legality-preserving choice-codec score bias with intervention telemetry" + }, + { + "version": "v11", + "date": "2026-07-20", + "note": "add SLM-138 shared recursive denoiser tower wiring, deep-supervision plumbing, and migration helper" + }, + { + "version": "v10", + "date": "2026-07-20", + "note": "expose bounded rare slot-owner record sampling in the shared TwoTower training recipe" + }, + { + "version": "v9", + "date": "2026-07-20", + "note": "use a masked non-empty sentinel for terminal slot-pair context so HF tokenization cannot produce a zero-length sequence" + }, + { + "version": "v8", + "date": "2026-07-19", + "note": "smooth slot lexeme-owner priors with class-prior-proportional pseudo-counts so unseen token-class pairs cannot receive spurious positive association" + }, + { + "version": "v7", + "date": "2026-07-19", + "note": "expose the bounded strict-subset root-reference identity sampling multiplier in the shared model-build recipe" + }, + { + "version": "v6", + "date": "2026-07-19", + "note": "add bounded excluded-reference weighting and negative-class accuracy telemetry to terminal-root identity supervision" + }, + { + "version": "v5", + "date": "2026-07-19", + "note": "permute existing terminal-root reference scores by learned identity rank while preserving the reference-group maximum and leaving non-reference candidates unchanged" + }, + { + "version": "v4", + "date": "2026-07-19", + "note": "center terminal-root identity bias over unused legal references so identity ranks references without changing the arity head's continue-versus-stop partition" + }, + { + "version": "v3", + "date": "2026-07-19", + "note": "add bounded terminal-root reference identity supervision and learned legal-reference inclusion bias" + }, + { + "version": "v2", + "date": "2026-07-19", + "note": "bound terminal-root reference-arity targets and auxiliary loss classes by each generated stream's available sections" + }, + { + "version": "v1", + "date": "2026-07-19", + "note": "initial registration; SLM-123 removable low-rank delta adapter backend, factory/config integration, and CLI wiring" + } + ] + }, + "harness.experiments.slm192_profile_flow_pipeline": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm192_profile_flow_pipeline.py", + "tests/test_harnesses/experiments/test_slm192_profile_flow_pipeline.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-192 FFE3-01 stage-accurate cost profiler for valid-edit bridge, training, decode, closure, and verification" + } + ] + }, + "matrix.slm192_profile_flow_pipeline": { + "version": "ffe3-01-v1", + "kind": "matrix", + "paths": [ + "scripts/profile_flow_pipeline.py", + "tests/test_scripts/test_profile_flow_pipeline.py", + "docs/design/iter-slm192-profile-flow-pipeline-20260721.json", + "docs/design/iter-slm192-profile-flow-pipeline-20260721.md" + ], + "history": [ + { + "version": "ffe3-01-v1", + "date": "2026-07-21", + "note": "initial registration; SLM-192 FFE3-01 flow-pipeline cost-profile fixture matrix CLI" + } + ] + }, + "harness.experiments.slm193_flow_caches": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm193_flow_caches.py", + "src/slm_training/harness_core/flow_cache.py", + "tests/test_harnesses/experiments/test_slm193_flow_caches.py", + "tests/test_harness_core/test_flow_cache.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-21", + "note": "initial registration; SLM-193 FFE3-02 bit-exact state, candidate, closure, and bridge cache fixture harness" + } + ] + }, + "matrix.slm193_flow_caches": { + "version": "ffe3-02-v1", + "kind": "matrix", + "paths": [ + "scripts/bench_flow_caches.py", + "tests/test_scripts/test_bench_flow_caches.py", + "docs/design/iter-slm193-flow-caches-20260721.json", + "docs/design/iter-slm193-flow-caches-20260721.md" + ], + "history": [ + { + "version": "ffe3-02-v1", + "date": "2026-07-21", + "note": "no-bump: regenerate SLM-193 design artifact timings after githook fixture run; matrix CLI behavior unchanged" + }, + { + "version": "ffe3-02-v1", + "date": "2026-07-21", + "note": "initial registration; SLM-193 FFE3-02 bit-exact flow-cache fixture matrix CLI" + } + ] + }, + "flow.proposals": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/flow/proposals.py", + "tests/test_flow/test_proposals.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial registration; legality-neutral dynamic candidate proposal prefixes with mandatory exact fallback" + } + ] + }, + "harness.experiments.slm194_candidate_proposals": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm194_candidate_proposals.py", + "tests/test_harnesses/experiments/test_slm194_candidate_proposals.py" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-23", + "note": "initial registration; SLM-194 FFE3-03 candidate proposal recall, fallback, parity, and wall-cost fixture" + } + ] + }, + "harness.experiments.valid_edit_flow_closeout": { + "version": "v3", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/valid_edit_flow_closeout.py", + "tests/test_harnesses/experiments/test_valid_edit_flow_closeout.py" + ], + "history": [ + { + "version": "v3", + "date": "2026-07-24", + "note": "expand the deterministic closeout to the required 18-lever causal table and explicit authorities" + }, + { + "version": "v2", + "date": "2026-07-24", + "note": "stamp the closeout with its dedicated harness and matrix component versions" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; fail-closed SLM-207 evidence lock and causal disposition" + } + ] + }, + "matrix.valid_edit_flow_closeout": { + "version": "vfa3-04-v3", + "kind": "matrix", + "paths": [ + "scripts/run_valid_edit_flow_closeout.py", + "tests/test_scripts/test_run_valid_edit_flow_closeout.py", + "docs/design/valid-edit-flow-closeout.json", + "docs/design/valid-edit-flow-closeout.md", + "docs/design/valid-edit-flow-selected-stack.json", + "docs/design/adr-valid-edit-flow-closeout.md", + "docs/design/valid-edit-flow-architecture.md" + ], + "history": [ + { + "version": "vfa3-04-v3", + "date": "2026-07-24", + "note": "emit ADR, selected-stack inventory, and hard-soft authority map with the complete causal table" + }, + { + "version": "vfa3-04-v2", + "date": "2026-07-24", + "note": "isolate design-output destination so closeout verification tests do not mutate committed evidence" + }, + { + "version": "vfa3-04-v1", + "date": "2026-07-24", + "note": "initial registration; deterministic VFA program closeout" + } + ] + }, + "matrix.slm194_candidate_proposals": { + "version": "ffe3-03-v1", + "kind": "matrix", + "paths": [ + "scripts/run_candidate_proposal_matrix.py", + "tests/test_scripts/test_run_candidate_proposal_matrix.py", + "docs/design/iter-slm194-candidate-proposals-20260724.json", + "docs/design/iter-slm194-candidate-proposals-20260724.md", + "docs/design/iter-slm194-candidate-proposals-agentv-20260724/", + "docs/design/perf-experiment-matrix.md" + ], + "history": [ + { + "version": "ffe3-03-v1", + "date": "2026-07-23", + "note": "no-bump: regenerate measured evidence after final prerequisite squash lineage without changing matrix behavior" + }, + { + "version": "ffe3-03-v1", + "date": "2026-07-23", + "note": "initial registration; SLM-194 FFE3-03 k-grid and true-wall proposal amortization matrix" + } + ] + }, + "harness.eval.replay": { + "version": "v3", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/eval/harness_replay.py", + "scripts/replay_harness_failures.py", + "tests/test_harnesses/eval/test_harness_replay.py", + "docs/design/harness-artifact-audit.md" + ], + "history": [ + { + "version": "v3", + "date": "2026-07-24", + "note": "Run deterministic five-suite local byte-reverification with production scoring, parser cascade, runtime evidence, and automatic evidence-only architecture blocking" + }, + { + "version": "v2", + "date": "2026-07-24", + "note": "Represent missing causal perturbation traces as unknown rather than fabricated zero label flips" + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; byte-preserving archived harness replay records" + } + ] + }, + "harness.experiments.slm310_action_alignment": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm310_action_alignment.py", + "scripts/run_slm310_action_alignment.py", + "tests/test_harnesses/experiments/test_slm310_action_alignment.py", + "docs/design/iter-slm310-action-alignment-20260724.json", + "docs/design/iter-slm310-action-alignment-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-310 LAR2-03 action alignment: reason-coded proposal telemetry, train-target vs decode-demand distribution audit, declared inverse-action corruption sampler, legacy/corrected STOP-slot accounting arms, matched 2x2 fixture experiment with preregistered thresholds" + } + ] + }, + "harness.experiments.slm312_state_sources": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm312_state_sources.py", + "scripts/run_slm312_state_sources.py", + "tests/test_harnesses/experiments/test_slm312_state_sources.py", + "docs/design/iter-slm312-state-sources-20260724.json", + "docs/design/iter-slm312-state-sources-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-312 LAR2-04 state sources: offline seedward oracle-guided valid intermediates + immutable content-addressed on-policy beam-trajectory snapshots labeled with the SLM-308 distance oracle and SLM-305 legal edits, fail-closed AST-fingerprint leak guards, matched {gold_only, seedward, on_policy, mixed} mixture arms with preregistered thresholds" + } + ] + }, + "harness.experiments.slm314_winner_take_all": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm314_winner_take_all.py", + "scripts/run_slm314_winner_take_all.py", + "tests/test_harnesses/experiments/test_slm314_winner_take_all.py", + "src/slm_training/resources/data/slm314_multimode/modes.jsonl", + "src/slm_training/resources/data/slm314_multimode/modes.jsonl.manifest.json", + "docs/design/iter-slm314-winner-take-all-20260724.json", + "docs/design/iter-slm314-winner-take-all-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-314 LAR2-05 winner-take-all (MCL): frozen 8-prompt x 2-mode verifier-accepted canonical-AST dataset (alpha-invariant fingerprints + sha256 manifest), min-loss-mode loss with preregistered 0.1 coverage/floor term, deterministic two-mode synthetic collapse-vs-retain proof, matched {single_gold, multi_gold, wta} arms" + } + ] + }, + "harness.experiments.slm317_repair_hybrid": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm317_repair_hybrid.py", + "scripts/run_slm317_repair_hybrid.py", + "tests/test_harnesses/experiments/test_slm317_repair_hybrid.py", + "docs/design/iter-slm317-repair-hybrid-20260724.json", + "docs/design/iter-slm317-repair-hybrid-20260724.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-317 LAR2-06 do-no-harm AR->repair hybrid: metamorphism-invariant commit rule (hard ladder + calibrated soft score), matched arms incl. historical/improved repair + oracle upper bound, invalid-over-valid visibility, advancement gates and repair_positive|repair_negative|inconclusive disposition" + } + ] + }, + "harness.experiments.slm338_evidence_publisher": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/autoresearch/", + "scripts/publish_planning_result.py", + "tests/test_harnesses/autoresearch/", + "tests/test_scripts/test_publish_planning_result.py", + "docs/design/abstract-planning-evidence-contract.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-25", + "note": "Add SLM-338/AP-037 unified abstract-planning evidence publisher (AbstractPlanningResultV1, fail-closed publication blockers, JSON/MD/model-card emission)" + } + ] } + } } diff --git a/tests/fixtures/planning_result/ap037_campaign.json b/tests/fixtures/planning_result/ap037_campaign.json new file mode 100644 index 000000000..da498d253 --- /dev/null +++ b/tests/fixtures/planning_result/ap037_campaign.json @@ -0,0 +1,89 @@ +{ + "arms": [ + { + "arm_id": "control", + "config_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "role": "control" + }, + { + "arm_id": "candidate", + "config_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "role": "candidate" + } + ], + "artifact_requirements": [ + { + "kind": "version_stamp", + "minimum_count": 1 + } + ], + "author": "slm338-fixture", + "budget": { + "max_experiments": 1, + "max_gpu_hours": 0.0, + "max_wall_minutes": 2.0 + }, + "campaign_id": "ap-037-fixture", + "claim_class": "diagnostic", + "controls": [ + { + "control_id": "random-plan", + "description": "Random plan must underperform.", + "kind": "negative" + } + ], + "created_at": "2026-07-25T00:00:00Z", + "decision": "Diagnose only; no promotion claim from fixture scale.", + "endpoints": [ + { + "direction": "increase", + "endpoint_id": "meaning", + "metric": "binder_reference_f1", + "minimum_effect": 0.01, + "role": "primary" + } + ], + "experiment_id": "e001", + "hypothesis": "Candidate planning improves the locked primary endpoint.", + "multiplicity_families": [ + { + "alpha": 0.05, + "family_id": "primary", + "hypothesis_ids": [ + "meaning" + ], + "method": "holm" + } + ], + "negative_controls": [ + "random-plan" + ], + "promotion_gates": [ + { + "endpoint_id": "meaning", + "gate_id": "meaning-improves", + "operator": "ge", + "threshold": 0.01 + } + ], + "requires_rl": false, + "rl_evaluation_sha256": null, + "rl_readiness_report_sha256": null, + "rollback_gates": [ + { + "endpoint_id": "meaning", + "gate_id": "meaning-regresses", + "operator": "le", + "threshold": -0.01 + } + ], + "schema_version": "ExperimentCampaignV1", + "seeds": [ + 7 + ], + "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "source_dirty": false, + "stopping_rules": [ + "Stop after the declared seed finishes." + ] +} diff --git a/tests/fixtures/planning_result/ap037_fixture.json b/tests/fixtures/planning_result/ap037_fixture.json new file mode 100644 index 000000000..b362de534 --- /dev/null +++ b/tests/fixtures/planning_result/ap037_fixture.json @@ -0,0 +1,124 @@ +{ + "agentv": { + "calibration_version": "agentv-cal-v3", + "n": 24, + "path": "outputs/ap037/agentv.jsonl" + }, + "campaign_id": "ap-037-fixture", + "campaign_manifest_sha256": "00601c6d1c08dbe7ca22a3402f2c9e93d9178d9634018e0c6db29baef3215f41", + "checkpoint": { + "name": "ap037_fixture.pt", + "sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "size_bytes": 1024, + "uri": "outputs/runs/ap037_fixture/checkpoints/last.pt" + }, + "claim_class": "diagnostic", + "controls": [ + { + "description": "Gold plan oracle ceiling.", + "kind": "oracle", + "metrics": { + "binder_reference_f1": 0.9583 + }, + "n": 24 + }, + { + "description": "Random plan negative control.", + "kind": "random", + "metrics": { + "binder_reference_f1": 0.2083 + }, + "n": 24 + }, + { + "description": "Empty plan negative control.", + "kind": "empty", + "metrics": { + "binder_reference_f1": 0.1667 + }, + "n": 24 + }, + { + "description": "Shuffled step-order negative control.", + "kind": "shuffled", + "metrics": { + "binder_reference_f1": 0.2292 + }, + "n": 24 + } + ], + "data_snapshot_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "human_audit": { + "calibration_version": "human-cal-v1", + "n": 12, + "path": "outputs/ap037/human_audit.md" + }, + "interventions": [ + { + "delta": -0.4375, + "description": "Shuffle plan step order", + "intervention_id": "shuffle-steps", + "metric": "binder_reference_f1" + } + ], + "latency": { + "generation_seconds": 41.5, + "p95_seconds": 58.2, + "plan_seconds": 3.25, + "total_seconds": 51.5, + "verification_seconds": 6.75 + }, + "metrics": [ + { + "artifact": { + "name": "raw.jsonl", + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "size_bytes": 2048, + "uri": "outputs/ap037/raw.jsonl" + }, + "binder_reference_f1": 0.4583, + "meaning_v2": 0.3125, + "n": 24, + "parse_rate": 0.625, + "path": "raw" + }, + { + "artifact": { + "name": "constrained.jsonl", + "sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "size_bytes": 2048, + "uri": "outputs/ap037/constrained.jsonl" + }, + "binder_reference_f1": 0.6667, + "meaning_v2": 0.5417, + "n": 24, + "parse_rate": 1.0, + "path": "constrained" + }, + { + "artifact": { + "name": "repaired.jsonl", + "sha256": "3333333333333333333333333333333333333333333333333333333333333333", + "size_bytes": 2048, + "uri": "outputs/ap037/repaired.jsonl" + }, + "binder_reference_f1": 0.7083, + "meaning_v2": 0.5833, + "n": 24, + "parse_rate": 1.0, + "path": "repaired" + } + ], + "schema": "AbstractPlanningResultV1", + "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "source_dirty": false, + "verifier_gates": [ + { + "gate_id": "meaning-improves", + "observed": 0.2084, + "passed": true, + "threshold": 0.01 + } + ], + "version_stamp": {} +} diff --git a/tests/test_harnesses/autoresearch/__init__.py b/tests/test_harnesses/autoresearch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_harnesses/autoresearch/test_planning_result.py b/tests/test_harnesses/autoresearch/test_planning_result.py new file mode 100644 index 000000000..556ac0a3e --- /dev/null +++ b/tests/test_harnesses/autoresearch/test_planning_result.py @@ -0,0 +1,148 @@ +"""Contract tests for the AP-037 / SLM-338 abstract-planning result object.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +from slm_training.autoresearch.experiment_campaign import ( + ExperimentCampaignV1, + campaign_manifest_sha256, +) +from slm_training.harnesses.autoresearch.planning_result import ( + SCHEMA_ID, + AbstractPlanningResultV1, + LatencyBreakdownV1, + MetricPathV1, + PlanControlV1, + publication_blockers, +) +from slm_training.harness_core.evidence_bundle import ArtifactRef + +FIXTURE = ( + Path(__file__).resolve().parents[2] + / "fixtures" + / "planning_result" + / "ap037_fixture.json" +) +HEX_64 = "b" * 64 + + +def _fixture_payload() -> dict: + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +def _result(**updates) -> AbstractPlanningResultV1: + result = AbstractPlanningResultV1.from_dict(_fixture_payload()) + return replace(result, **updates) + + +def test_fixture_roundtrips_and_content_sha_is_deterministic() -> None: + result = AbstractPlanningResultV1.from_dict(_fixture_payload()) + assert result.schema == SCHEMA_ID + again = AbstractPlanningResultV1.from_dict(result.to_dict()) + assert again == result + assert result.content_sha256() == again.content_sha256() + assert len(result.content_sha256()) == 64 + + +def test_fixture_manifest_hash_matches_locked_campaign() -> None: + payload = _fixture_payload() + assert payload["campaign_manifest_sha256"] == ( + "00601c6d1c08dbe7ca22a3402f2c9e93d9178d9634018e0c6db29baef3215f41" + ) + # The digest function is the canonical campaign lock hash from SLM-337. + manifest = ExperimentCampaignV1.model_validate( + json.loads(FIXTURE.with_name("ap037_campaign.json").read_text()) + ) + assert campaign_manifest_sha256(manifest) == payload["campaign_manifest_sha256"] + + +def test_valid_diagnostic_result_has_no_blockers() -> None: + assert publication_blockers(_result()) == [] + + +@pytest.mark.parametrize( + ("field", "value", "reason"), + [ + ("campaign_manifest_sha256", "", "missing_campaign_manifest_sha256"), + ("campaign_manifest_sha256", "not-a-sha", "missing_campaign_manifest_sha256"), + ("source_commit", "UNKNOWN", "missing_provenance:source_commit"), + ("source_commit", "", "missing_provenance:source_commit"), + ("data_snapshot_sha256", "UNKNOWN", "missing_provenance:data_snapshot_sha256"), + ], +) +def test_missing_provenance_and_manifest_block( + field: str, value: str, reason: str +) -> None: + blockers = publication_blockers(_result(**{field: value})) + assert reason in blockers + + +def test_missing_total_latency_blocks() -> None: + latency = replace(_result().latency, total_seconds=None) + assert "missing_total_latency" in publication_blockers(_result(latency=latency)) + latency = replace(_result().latency, total_seconds=0.0) + assert "missing_total_latency" in publication_blockers(_result(latency=latency)) + + +def test_promotion_classes_require_controls_paths_and_checkpoint() -> None: + promotion = _result(claim_class="promotion_candidate") + assert publication_blockers(promotion) == [] + + no_controls = replace(promotion, controls=()) + assert "missing_plan_controls" in publication_blockers(no_controls) + + raw_only = replace(promotion, metrics=(promotion.metric_path("raw"),)) + blockers = publication_blockers(raw_only) + assert "missing_metric_paths:constrained,repaired" in blockers + + no_checkpoint = replace(promotion, checkpoint=None) + assert "missing_provenance:checkpoint" in publication_blockers(no_checkpoint) + + +def test_diagnostic_class_does_not_require_controls_or_full_paths() -> None: + diagnostic = _result( + claim_class="diagnostic", + controls=(), + checkpoint=None, + metrics=(MetricPathV1(path="raw", n=4, parse_rate=0.5),), + ) + assert publication_blockers(diagnostic) == [] + + +def test_ship_gate_class_is_treated_as_promotion() -> None: + ship = _result(claim_class="ship_gate", controls=()) + assert "missing_plan_controls" in publication_blockers(ship) + + +def test_unknown_claim_class_blocks() -> None: + blockers = publication_blockers(_result(claim_class="bogus")) + assert "unknown_claim_class:bogus" in blockers + + +def test_negative_control_kinds_satisfy_controls() -> None: + promotion = _result( + claim_class="promotion_candidate", + controls=(PlanControlV1(kind="shuffled", n=8),), + ) + assert "missing_plan_controls" not in publication_blockers(promotion) + oracle_only = _result( + claim_class="promotion_candidate", + controls=(PlanControlV1(kind="oracle", n=8),), + ) + assert "missing_plan_controls" in publication_blockers(oracle_only) + + +def test_latency_default_is_blocked_not_silent() -> None: + result = _result(latency=LatencyBreakdownV1()) + assert "missing_total_latency" in publication_blockers(result) + + +def test_checkpoint_artifact_ref_roundtrip() -> None: + ref = ArtifactRef(name="ckpt.pt", sha256=HEX_64, size_bytes=1, uri="outputs/x.pt") + result = _result(checkpoint=ref) + assert AbstractPlanningResultV1.from_dict(result.to_dict()).checkpoint == ref diff --git a/tests/test_scripts/test_publish_planning_result.py b/tests/test_scripts/test_publish_planning_result.py new file mode 100644 index 000000000..5a7c1cf84 --- /dev/null +++ b/tests/test_scripts/test_publish_planning_result.py @@ -0,0 +1,120 @@ +"""End-to-end tests for scripts/publish_planning_result.py (AP-037 / SLM-338).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from scripts.publish_planning_result import main +from slm_training.harnesses.autoresearch.planning_result import ( + AbstractPlanningResultV1, + render_markdown, +) + +FIXTURE = ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "planning_result" + / "ap037_fixture.json" +) +DOCS_DESIGN = Path(__file__).resolve().parents[2] / "docs" / "design" + + +def _publish(tmp_path: Path, *extra: str) -> int: + return main([str(FIXTURE), "--output", str(tmp_path / "evidence"), *extra]) + + +def test_fixture_publishes_json_md_and_snippet_from_one_object(tmp_path: Path) -> None: + snippet = tmp_path / "row.md" + assert _publish(tmp_path, "--snippet-out", str(snippet)) == 0 + + canonical = json.loads((tmp_path / "evidence.json").read_text(encoding="utf-8")) + markdown = (tmp_path / "evidence.md").read_text(encoding="utf-8") + row = snippet.read_text(encoding="utf-8") + + assert canonical["schema"] == "AbstractPlanningResultV1" + result = AbstractPlanningResultV1.from_dict(canonical) + assert canonical["content_sha256"] == result.content_sha256() + + # Every headline number in the Markdown comes from the JSON payload. + constrained = next(m for m in canonical["metrics"] if m["path"] == "constrained") + assert ( + f"| constrained | {constrained['n']} | {constrained['meaning_v2']:.4f} " + in markdown + ) + assert f"{canonical['latency']['total_seconds']:.4f}" in markdown + assert canonical["campaign_manifest_sha256"] in markdown + + # Model-card row is derived, not hand-written. + assert row.startswith("| ap-037-fixture abstract-planning result |") + assert "not promotable or ship" in row # diagnostic honesty tag + + +def test_markdown_is_a_deterministic_render_of_the_json(tmp_path: Path) -> None: + assert _publish(tmp_path) == 0 + canonical = json.loads((tmp_path / "evidence.json").read_text(encoding="utf-8")) + assert render_markdown(canonical) == (tmp_path / "evidence.md").read_text( + encoding="utf-8" + ) + + +def test_blockers_fail_closed_with_reasons(tmp_path: Path, capsys) -> None: + payload = json.loads(FIXTURE.read_text(encoding="utf-8")) + payload["source_commit"] = "UNKNOWN" + payload["latency"]["total_seconds"] = None + broken = tmp_path / "broken.json" + broken.write_text(json.dumps(payload), encoding="utf-8") + rc = main([str(broken), "--output", str(tmp_path / "nope")]) + assert rc == 1 + err = capsys.readouterr().err + assert "missing_provenance:source_commit" in err + assert "missing_total_latency" in err + assert not (tmp_path / "nope.json").exists() + + +def test_disposition_append_is_idempotent_per_campaign_hash(tmp_path: Path) -> None: + doc = tmp_path / "disposition.md" + assert _publish(tmp_path, "--append-disposition", str(doc)) == 0 + assert _publish(tmp_path, "--append-disposition", str(doc)) == 0 + text = doc.read_text(encoding="utf-8") + manifest_sha = json.loads(FIXTURE.read_text(encoding="utf-8"))[ + "campaign_manifest_sha256" + ] + assert text.count(f"## Campaign `{manifest_sha}`") == 1 + + +def test_from_dir_merges_harness_output_fragments(tmp_path: Path) -> None: + payload = json.loads(FIXTURE.read_text(encoding="utf-8")) + harness_dir = tmp_path / "harness_out" + harness_dir.mkdir() + (harness_dir / "10_result.json").write_text( + json.dumps( + {k: v for k, v in payload.items() if k not in {"metrics", "latency"}} + ) + ) + (harness_dir / "20_metrics.json").write_text( + json.dumps({"metrics": payload["metrics"], "latency": payload["latency"]}) + ) + rc = main([str(harness_dir), "--from-dir", "--output", str(tmp_path / "merged")]) + assert rc == 0 + canonical = json.loads((tmp_path / "merged.json").read_text(encoding="utf-8")) + assert len(canonical["metrics"]) == 3 + + +def test_publisher_writes_only_its_own_artifacts(tmp_path: Path) -> None: + doc = tmp_path / "disposition.md" + before = {p: p.read_bytes() for p in DOCS_DESIGN.glob("iter-*") if p.is_file()} + assert ( + _publish( + tmp_path, + "--snippet-out", + str(tmp_path / "row.md"), + "--append-disposition", + str(doc), + ) + == 0 + ) + after = {p: p.read_bytes() for p in DOCS_DESIGN.glob("iter-*") if p.is_file()} + assert before == after # historical iter docs untouched + written = {p.name for p in tmp_path.rglob("*") if p.is_file()} + assert written == {"evidence.json", "evidence.md", "row.md", "disposition.md"}