From f5d09bce50d493a7d7c0f3e60adbc65e491ccebb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 3 Sep 2026 07:46:51 -0400 Subject: [PATCH 01/10] Start notes route progress log --- PROGRESS.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 PROGRESS.md diff --git a/PROGRESS.md b/PROGRESS.md new file mode 100644 index 0000000..a752955 --- /dev/null +++ b/PROGRESS.md @@ -0,0 +1,19 @@ +# Progress + +## State + +Implementation is starting on branch `notes-route` from `d9a13ce`. + +## Done + +- Installed the app dependencies with `bun install --frozen-lockfile`. +- Confirmed the worktree is on `notes-route` at the requested starting commit. + +## Next + +- Inspect the frozen snapshot, existing route patterns, and PolicyEngine reference machinery. +- Add the SNAP pathway recompute script and commit its generated data. +- Add data-backed note definitions and Python tests. +- Add the `/notes` routes, navigation, scenario deep links, sitemap entries, and app tests. +- Run the requested Python and app verification suites. +- Write `out.md`, push `notes-route`, and open a draft PR against `add-fable-5-1`. From a305125ae72054d56b5fd995da25ef6ac4735477 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 3 Sep 2026 07:53:43 -0400 Subject: [PATCH 02/10] Add SNAP pathway recompute evidence --- .gitignore | 2 + PROGRESS.md | 9 +- notes/data/snap_pathways_20260901.csv | 101 ++++++++ .../data/snap_pathways_20260901.csv.meta.json | 6 + scripts/snap_pathways.py | 239 ++++++++++++++++++ 5 files changed, 354 insertions(+), 3 deletions(-) create mode 100644 notes/data/snap_pathways_20260901.csv create mode 100644 notes/data/snap_pathways_20260901.csv.meta.json create mode 100644 scripts/snap_pathways.py diff --git a/.gitignore b/.gitignore index e1b5f8b..a4b63cf 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ results/ground_truth.csv results/scenarios.csv output/ data/ +!notes/data/ +!notes/data/** # Committed sensitivity evidence (pinned by tests/test_sensitivity_evidence.py) !sensitivity/data/ diff --git a/PROGRESS.md b/PROGRESS.md index a752955..c163773 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,17 +2,20 @@ ## State -Implementation is starting on branch `notes-route` from `d9a13ce`. +The frozen SNAP pathways have been recomputed and are ready for note/test use. ## Done - Installed the app dependencies with `bun install --frozen-lockfile`. - Confirmed the worktree is on `notes-route` at the requested starting commit. +- Added `scripts/snap_pathways.py`, reusing the reference calculation's + vectorized situation builder and the installed PolicyEngine variables. +- Generated 100 pathway rows with policyengine-us 1.723.0. All recomputed SNAP + values match the frozen references exactly; 20 households receive SNAP and + 10 depend on categorical eligibility. ## Next -- Inspect the frozen snapshot, existing route patterns, and PolicyEngine reference machinery. -- Add the SNAP pathway recompute script and commit its generated data. - Add data-backed note definitions and Python tests. - Add the `/notes` routes, navigation, scenario deep links, sitemap entries, and app tests. - Run the requested Python and app verification suites. diff --git a/notes/data/snap_pathways_20260901.csv b/notes/data/snap_pathways_20260901.csv new file mode 100644 index 0000000..eba7180 --- /dev/null +++ b/notes/data/snap_pathways_20260901.csv @@ -0,0 +1,101 @@ +scenario_id,state,household_size,snap_reference,snap_recomputed,meets_gross_income_test,meets_net_income_test,meets_asset_test,is_tanf_non_cash_eligible,snap_eligible,pathway +scenario_000,TX,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_001,VA,2,0.0,0.0,True,False,True,False,False,ineligible +scenario_002,WA,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_003,TX,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_004,NY,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_005,CA,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_007,ID,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_008,NJ,8,15246.9052734375,15246.9052734375,True,True,False,True,True,categorical_assets +scenario_009,NC,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_012,MS,3,4952.08935546875,4952.08935546875,True,True,True,False,True,ordinary +scenario_013,AZ,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_014,WV,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_015,IN,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_016,FL,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_018,AZ,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_020,TX,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_021,MO,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_022,CA,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_023,CA,1,461.3397216796875,461.3397216796875,False,True,True,True,True,categorical_income +scenario_025,OH,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_026,NC,5,0.0,0.0,False,False,False,False,False,ineligible +scenario_027,CT,2,287.68316650390625,287.68316650390625,True,False,False,True,True,categorical_both +scenario_028,PA,4,0.0,0.0,False,False,True,True,False,ineligible +scenario_029,OK,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_030,TX,1,287.68316650390625,287.68316650390625,False,False,True,True,True,categorical_income +scenario_031,CA,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_032,MN,3,0.0,0.0,False,False,False,True,False,ineligible +scenario_033,SD,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_036,NJ,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_037,NC,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_038,LA,4,7286.9443359375,7286.9443359375,True,True,True,True,True,ordinary +scenario_039,VA,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_040,AZ,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_042,WI,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_043,CO,1,3596.039794921875,3596.039794921875,True,True,True,True,True,ordinary +scenario_044,KS,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_045,MI,1,287.68316650390625,287.68316650390625,False,True,True,True,True,categorical_income +scenario_046,OK,4,0.0,0.0,False,False,False,False,False,ineligible +scenario_048,OH,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_049,NH,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_051,LA,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_052,TX,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_053,ID,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_054,NC,3,6125.68896484375,6125.68896484375,True,True,False,True,True,categorical_assets +scenario_055,FL,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_056,NJ,1,1140.0,1140.0,True,True,True,True,True,ordinary +scenario_057,LA,2,2669.217041015625,2669.217041015625,True,True,True,True,True,ordinary +scenario_059,FL,2,0.0,0.0,True,False,True,False,False,ineligible +scenario_060,TX,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_062,FL,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_064,WI,5,0.0,0.0,False,False,False,False,False,ineligible +scenario_066,VA,1,3596.039794921875,3596.039794921875,True,True,False,True,True,categorical_assets +scenario_067,IN,3,0.0,0.0,True,False,False,False,False,ineligible +scenario_068,MD,1,0.0,0.0,False,True,True,False,False,ineligible +scenario_070,IL,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_071,NY,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_072,MI,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_073,MI,1,287.68316650390625,287.68316650390625,True,False,True,True,True,categorical_income +scenario_074,LA,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_075,FL,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_076,ID,3,0.0,0.0,False,False,False,False,False,ineligible +scenario_077,LA,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_078,MD,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_079,AZ,2,2428.017333984375,2428.017333984375,True,True,True,True,True,ordinary +scenario_080,PA,1,3596.039794921875,3596.039794921875,True,True,False,True,True,categorical_assets +scenario_081,MA,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_082,NY,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_083,TX,1,0.0,0.0,True,True,False,False,False,ineligible +scenario_084,NC,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_085,PA,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_086,GA,2,0.0,0.0,False,False,True,False,False,ineligible +scenario_088,TX,1,0.0,0.0,False,True,False,False,False,ineligible +scenario_089,NC,2,0.0,0.0,False,False,True,False,False,ineligible +scenario_090,KS,1,0.0,0.0,True,True,False,False,False,ineligible +scenario_091,WI,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_092,AL,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_093,MO,3,0.0,0.0,False,False,False,False,False,ineligible +scenario_095,NJ,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_098,IL,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_099,CA,4,0.0,0.0,False,False,False,False,False,ineligible +scenario_100,MT,3,8625.8896484375,8625.8896484375,True,True,True,True,True,ordinary +scenario_101,TX,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_102,NC,2,0.0,0.0,False,False,True,False,False,ineligible +scenario_104,NY,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_107,OH,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_108,WI,1,287.68316650390625,287.68316650390625,True,False,True,True,True,categorical_income +scenario_109,FL,5,8020.55419921875,8020.55419921875,True,True,True,True,True,ordinary +scenario_110,OH,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_111,WA,2,0.0,0.0,True,False,True,False,False,ineligible +scenario_112,TX,1,287.68316650390625,287.68316650390625,True,True,True,True,True,ordinary +scenario_114,VA,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_115,AL,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_116,FL,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_117,AR,5,0.0,0.0,False,False,False,False,False,ineligible +scenario_118,NY,1,2903.9404296875,2903.9404296875,True,True,True,True,True,ordinary +scenario_119,VA,3,0.0,0.0,False,False,False,False,False,ineligible +scenario_120,CT,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_121,SC,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_122,MN,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_123,PA,3,0.0,0.0,False,False,False,False,False,ineligible diff --git a/notes/data/snap_pathways_20260901.csv.meta.json b/notes/data/snap_pathways_20260901.csv.meta.json new file mode 100644 index 0000000..a044852 --- /dev/null +++ b/notes/data/snap_pathways_20260901.csv.meta.json @@ -0,0 +1,6 @@ +{ + "policyengine_us_version": "1.723.0", + "generated_at_utc": "2026-09-03T11:52:27.778906+00:00", + "scenarios_sha256": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", + "script": "scripts/snap_pathways.py" +} diff --git a/scripts/snap_pathways.py b/scripts/snap_pathways.py new file mode 100644 index 0000000..1fb1ac1 --- /dev/null +++ b/scripts/snap_pathways.py @@ -0,0 +1,239 @@ +"""Recompute SNAP eligibility pathways for the frozen 100-household snapshot.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from importlib.metadata import version +from pathlib import Path + +import numpy as np +import pandas as pd + +from policybench.ground_truth import ( + _build_us_vectorized_situation, + _extract_us_vectorized_value, +) +from policybench.policyengine_runtime import get_us_situation_simulation_class +from policybench.scenarios import load_scenarios_from_manifest + +ROOT = Path(__file__).resolve().parents[1] +RUN_DIR = ( + ROOT / "paper/snapshot/20260501/runs/" + "us_full_run_20260612_policyengine_4_16_1_populace" +) +SCENARIOS_PATH = RUN_DIR / "scenarios.csv" +REFERENCE_PATH = RUN_DIR / "reference_outputs.csv" +OUTPUT_PATH = ROOT / "notes/data/snap_pathways_20260901.csv" +META_PATH = OUTPUT_PATH.with_suffix(OUTPUT_PATH.suffix + ".meta.json") +SCRIPT_PATH = "scripts/snap_pathways.py" +YEAR = 2026 +MAX_REFERENCE_DIFFERENCE = 1.0 + +# policyengine-us 1.723.0 calls the underlying eligibility variable +# ``is_snap_eligible``. The output column keeps the requested shorter name and +# means a household has a positive computed allotment, matching how the note +# counts households that qualify in the frozen reference. The model-level gate +# is still calculated and checked while deriving each row. +VARIABLES = ( + "snap", + "meets_snap_gross_income_test", + "meets_snap_net_income_test", + "meets_snap_asset_test", + "is_tanf_non_cash_eligible", + "is_snap_eligible", +) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _bool(value: float) -> bool: + if value not in (0, 1): + raise ValueError(f"Expected a Boolean calculation, received {value!r}.") + return bool(value) + + +def _pathway( + *, + snap_eligible: bool, + gross: bool, + net: bool, + assets: bool, + categorical: bool, +) -> str: + if not snap_eligible: + return "ineligible" + if gross and net and assets: + return "ordinary" + if not categorical: + raise ValueError( + "A household with a positive SNAP allotment failed an ordinary " + "eligibility test without categorical eligibility." + ) + income_failure = not (gross and net) + if income_failure and not assets: + return "categorical_both" + if income_failure: + return "categorical_income" + if not assets: + return "categorical_assets" + raise AssertionError("Unreachable SNAP pathway combination.") + + +def _snap_references() -> pd.Series: + references = pd.read_csv(REFERENCE_PATH) + snap = references.loc[references["variable"].eq("snap")].copy() + if len(snap) != 100 or snap["scenario_id"].duplicated().any(): + raise ValueError( + "Expected exactly one frozen SNAP reference for each of 100 scenarios." + ) + return snap.set_index("scenario_id")["value"] + + +def build_rows() -> list[dict[str, object]]: + """Calculate pathway inputs with the same vectorized builder as references.""" + scenarios = load_scenarios_from_manifest(SCENARIOS_PATH) + if len(scenarios) != 100: + raise ValueError(f"Expected 100 frozen scenarios, found {len(scenarios)}.") + scenario_years = {scenario.year for scenario in scenarios} + if scenario_years != {YEAR}: + raise ValueError(f"Expected only {YEAR} scenarios, found {scenario_years}.") + + situation, scenario_indexes = _build_us_vectorized_situation(scenarios) + simulation_class = get_us_situation_simulation_class() + simulation = simulation_class(situation=situation) + + missing = [ + variable + for variable in VARIABLES + if variable not in simulation.tax_benefit_system.variables + ] + if missing: + raise ValueError(f"Installed policyengine-us is missing variables: {missing}") + + calculations: dict[str, tuple[np.ndarray, str]] = {} + for variable in VARIABLES: + entity_key = simulation.tax_benefit_system.variables[variable].entity.key + calculations[variable] = ( + np.asarray(simulation.calculate(variable, YEAR)), + entity_key, + ) + + references = _snap_references() + scenario_ids = {scenario.id for scenario in scenarios} + if scenario_ids != set(references.index): + raise ValueError("Frozen scenario and SNAP reference ids do not match.") + + rows: list[dict[str, object]] = [] + engine_eligible_without_allotment: list[str] = [] + for scenario in scenarios: + index = scenario_indexes[scenario.id] + + def result(variable: str) -> float: + values, entity_key = calculations[variable] + return _extract_us_vectorized_value( + values, + scenario=scenario, + variable=variable, + entity_key=entity_key, + index=index, + ) + + snap_reference = float(references.loc[scenario.id]) + snap_recomputed = result("snap") + if not np.isfinite([snap_reference, snap_recomputed]).all(): + raise ValueError(f"{scenario.id} has a non-finite SNAP value.") + gross = _bool(result("meets_snap_gross_income_test")) + net = _bool(result("meets_snap_net_income_test")) + assets = _bool(result("meets_snap_asset_test")) + categorical = _bool(result("is_tanf_non_cash_eligible")) + engine_eligible = _bool(result("is_snap_eligible")) + snap_eligible = snap_recomputed > 0 + if snap_eligible and not engine_eligible: + raise ValueError( + f"{scenario.id} has a positive SNAP allotment but fails " + "is_snap_eligible." + ) + if engine_eligible and not snap_eligible: + engine_eligible_without_allotment.append(scenario.id) + + rows.append( + { + "scenario_id": scenario.id, + "state": scenario.state, + "household_size": len(scenario.all_people), + "snap_reference": snap_reference, + "snap_recomputed": snap_recomputed, + "meets_gross_income_test": gross, + "meets_net_income_test": net, + "meets_asset_test": assets, + "is_tanf_non_cash_eligible": categorical, + "snap_eligible": snap_eligible, + "pathway": _pathway( + snap_eligible=snap_eligible, + gross=gross, + net=net, + assets=assets, + categorical=categorical, + ), + } + ) + + differences = [ + row + for row in rows + if abs(float(row["snap_recomputed"]) - float(row["snap_reference"])) + > MAX_REFERENCE_DIFFERENCE + ] + if differences: + details = "\n".join( + f" {row['scenario_id']}: reference={row['snap_reference']}, " + f"recomputed={row['snap_recomputed']}" + for row in differences + ) + raise RuntimeError( + "Recomputed SNAP differs from the frozen reference by more than " + f"${MAX_REFERENCE_DIFFERENCE:.0f}:\n{details}" + ) + + if engine_eligible_without_allotment: + print( + "PolicyEngine's is_snap_eligible gate is true but the computed " + "allotment is $0 for: " + ", ".join(engine_eligible_without_allotment) + ) + return rows + + +def main() -> None: + rows = build_rows() + OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame(rows).to_csv(OUTPUT_PATH, index=False) + metadata = { + "policyengine_us_version": version("policyengine-us"), + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "scenarios_sha256": _sha256(SCENARIOS_PATH), + "script": SCRIPT_PATH, + } + META_PATH.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + + pathway_counts = pd.Series(row["pathway"] for row in rows).value_counts() + print(f"Wrote {len(rows)} rows to {OUTPUT_PATH.relative_to(ROOT)}") + for pathway in ( + "ordinary", + "categorical_income", + "categorical_assets", + "categorical_both", + "ineligible", + ): + print(f" {pathway}: {int(pathway_counts.get(pathway, 0))}") + + +if __name__ == "__main__": + main() From d3280f3774cc29bea3658fe2f67900e94955f9e2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 3 Sep 2026 07:59:10 -0400 Subject: [PATCH 03/10] Add data-backed board notes --- PROGRESS.md | 7 +- .../2026-09-01-claude-fable-5-1-added.json | 47 ++++ .../notes/2026-09-03-six-snap-households.json | 100 +++++++ app/src/notes/index.ts | 30 +++ tests/test_notes.py | 245 ++++++++++++++++++ 5 files changed, 427 insertions(+), 2 deletions(-) create mode 100644 app/src/notes/2026-09-01-claude-fable-5-1-added.json create mode 100644 app/src/notes/2026-09-03-six-snap-households.json create mode 100644 app/src/notes/index.ts create mode 100644 tests/test_notes.py diff --git a/PROGRESS.md b/PROGRESS.md index c163773..9d1df6a 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,7 +2,7 @@ ## State -The frozen SNAP pathways have been recomputed and are ready for note/test use. +Both notes and their frozen-evidence fact tests are implemented. ## Done @@ -13,10 +13,13 @@ The frozen SNAP pathways have been recomputed and are ready for note/test use. - Generated 100 pathway rows with policyengine-us 1.723.0. All recomputed SNAP values match the frozen references exactly; 20 households receive SNAP and 10 depend on categorical eligibility. +- Added the two dated note JSON files and a newest-first TypeScript index. +- Added Python coverage that derives every placeholder fact, the explanation + mention counts, categorical pathway counts, asset-case error bands, and + engine versions from committed evidence. The focused suite passes (5 tests). ## Next -- Add data-backed note definitions and Python tests. - Add the `/notes` routes, navigation, scenario deep links, sitemap entries, and app tests. - Run the requested Python and app verification suites. - Write `out.md`, push `notes-route`, and open a draft PR against `add-fable-5-1`. diff --git a/app/src/notes/2026-09-01-claude-fable-5-1-added.json b/app/src/notes/2026-09-01-claude-fable-5-1-added.json new file mode 100644 index 0000000..fabea8f --- /dev/null +++ b/app/src/notes/2026-09-01-claude-fable-5-1-added.json @@ -0,0 +1,47 @@ +{ + "slug": "2026-09-01-claude-fable-5-1-added", + "date": "2026-09-01", + "title": "Claude Fable 5.1 added", + "paragraphs": [ + "Claude Fable 5.1 was released on 2026-09-01 and joined the board the same day. Board row: {exactRate}% of answers within $1, #{rank} of {nModels}. {parsed} of {answers} answers parsed.", + "The model's API rejects forced tool calls, which is how the board collects most models' answers, so its row answers as a JSON object and reasons at the provider default. The serving-configuration table in the paper records the transport for every row.", + "Sensitivity: with the answer tool declared under tool_choice auto, Fable 5.1 scores {autoRate}% (would rank #{autoRank}). Under the same request Claude Fable 5 scores {fable5AutoRate}%. Fable 5's board row ({fable5BoardRate}%) ran chunked under a forced tool call, which switches off its thinking, so the {boardGap}-point gap between the two board rows is mostly request shape.", + "The next board version moves every model to tool_choice auto so each provider's default reasoning engages under the recorded request shape (policybench#139)." + ], + "facts": { + "exactRate": 86.3, + "rank": 2, + "nModels": 33, + "parsed": 1984, + "answers": 1984, + "autoRate": 87.5, + "autoRank": 2, + "fable5AutoRate": 86.9, + "fable5BoardRate": 79.9, + "boardGap": 6.4 + }, + "data": [ + { + "label": "Dashboard data release", + "href": "https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20260901c" + }, + { + "label": "Claude thinking sensitivity note", + "href": "https://github.com/PolicyEngine/policybench/blob/main/sensitivity/claude-thinking-2026-08.md" + }, + { + "label": "Pinned Fable 5.1 sensitivity evidence", + "href": "https://github.com/PolicyEngine/policybench/blob/main/sensitivity/data/claude-fable-5-1-thinking.json" + }, + { + "label": "Paper serving-configuration table", + "href": "/paper" + }, + { + "label": "Claude Fable 5.1 model page", + "href": "/model/claude-fable-5.1" + } + ], + "boardSnapshot": "2026-09-01", + "release": "dashboard-data-20260901c" +} diff --git a/app/src/notes/2026-09-03-six-snap-households.json b/app/src/notes/2026-09-03-six-snap-households.json new file mode 100644 index 0000000..8f983cf --- /dev/null +++ b/app/src/notes/2026-09-03-six-snap-households.json @@ -0,0 +1,100 @@ +{ + "slug": "2026-09-03-six-snap-households", + "date": "2026-09-03", + "title": "Six SNAP households the top three models deny", + "paragraphs": [ + "{eligibleCount} of the 100 households qualify for SNAP in the PolicyEngine reference. The top three models on the board, GPT-5.6 Sol, Claude Fable 5.1, and Kimi K3, each predict $0 for the same {deniedCount}: {deniedScenarios}. The reference for each is ${referenceAnnual} for the year, the minimum allotment (8% of the one-person maximum, ${referenceMonthly} a month, for households of one or two).", + "Five of the six qualify through broad-based categorical eligibility: Connecticut, Michigan, Texas, and Wisconsin confer SNAP eligibility on households receiving a TANF-funded non-cash benefit, with gross-income limits of 200% of poverty (CT, MI, WI) and 165% (TX) and the net-income and asset tests waived. The sixth, scenario_112 in Texas, passes the federal tests; the models counted farm-rent income the rules exclude.", + "{categoricalOnlyCount} of the {eligibleCount} eligible households qualify only through categorical eligibility: {categoricalIncomeCount} because income exceeds the federal limits, {categoricalAssetCount} because savings exceed the federal asset limit. On the {categoricalAssetCount} asset cases the models compute benefits near the reference: Sol within 1% on three and 10% on one, Fable 5.1 within 1% on all four.", + "In their SNAP explanations across the 100 households, Sol mentions categorical eligibility {solCategoricalMentions} time(s) and assets {solAssetMentions} times; Fable 5.1 mentions broad-based categorical eligibility {fableBbceMentions} times; Kimi K3 mentions categorical eligibility {kimiCategoricalMentions} times.", + "Pathways were recomputed with policyengine-us {pathwayEngineVersion} from the frozen scenarios and agree with the frozen reference SNAP values for all 100 households; the reference itself was generated with policyengine-us {referenceEngineVersion}." + ], + "facts": { + "eligibleCount": 20, + "deniedCount": 6, + "deniedScenarios": [ + "scenario_027", + "scenario_030", + "scenario_045", + "scenario_073", + "scenario_108", + "scenario_112" + ], + "referenceAnnual": 287.68, + "referenceMonthly": 23.84, + "categoricalOnlyCount": 10, + "categoricalIncomeCount": 6, + "categoricalAssetCount": 4, + "solCategoricalMentions": 1, + "solAssetMentions": 36, + "fableBbceMentions": 28, + "kimiCategoricalMentions": 17, + "pathwayEngineVersion": "1.723.0", + "referenceEngineVersion": "1.755.4" + }, + "mentionRegexes": { + "categorical": "categorical", + "assets": "asset|resource", + "bbce": "broad-based|bbce" + }, + "data": [ + { + "label": "scenario_027", + "href": "/?country=us&scenario=scenario_027#scenarios" + }, + { + "label": "scenario_030", + "href": "/?country=us&scenario=scenario_030#scenarios" + }, + { + "label": "scenario_045", + "href": "/?country=us&scenario=scenario_045#scenarios" + }, + { + "label": "scenario_073", + "href": "/?country=us&scenario=scenario_073#scenarios" + }, + { + "label": "scenario_108", + "href": "/?country=us&scenario=scenario_108#scenarios" + }, + { + "label": "scenario_112", + "href": "/?country=us&scenario=scenario_112#scenarios" + }, + { + "label": "Release predictions.csv.gz", + "href": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260901c/predictions.csv.gz" + }, + { + "label": "Release dashboard-data.json", + "href": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260901c/dashboard-data.json" + }, + { + "label": "Frozen predictions.csv.gz", + "href": "https://github.com/PolicyEngine/policybench/blob/main/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/predictions.csv.gz" + }, + { + "label": "Frozen reference_outputs.csv", + "href": "https://github.com/PolicyEngine/policybench/blob/main/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_outputs.csv" + }, + { + "label": "Frozen scenarios.csv", + "href": "https://github.com/PolicyEngine/policybench/blob/main/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/scenarios.csv" + }, + { + "label": "Recomputed SNAP pathways", + "href": "https://github.com/PolicyEngine/policybench/blob/main/notes/data/snap_pathways_20260901.csv" + }, + { + "label": "TANF non-cash parameters", + "href": "https://github.com/PolicyEngine/policyengine-us/tree/master/policyengine_us/parameters/gov/hhs/tanf/non_cash" + }, + { + "label": "SNAP minimum-allotment parameters", + "href": "https://github.com/PolicyEngine/policyengine-us/tree/master/policyengine_us/parameters/gov/usda/snap/min_allotment" + } + ], + "boardSnapshot": "2026-09-01", + "release": "dashboard-data-20260901c" +} diff --git a/app/src/notes/index.ts b/app/src/notes/index.ts new file mode 100644 index 0000000..60cccb2 --- /dev/null +++ b/app/src/notes/index.ts @@ -0,0 +1,30 @@ +import claudeFable51Added from "./2026-09-01-claude-fable-5-1-added.json"; +import sixSnapHouseholds from "./2026-09-03-six-snap-households.json"; + +export type NoteFact = number | string | string[]; + +export type NoteDataLink = { + label: string; + href: string; +}; + +export type PolicyBenchNote = { + slug: string; + date: string; + title: string; + paragraphs: string[]; + facts: Record; + mentionRegexes?: Record; + data: NoteDataLink[]; + boardSnapshot: "2026-09-01"; + release: "dashboard-data-20260901c"; +}; + +export const notes = [ + sixSnapHouseholds, + claudeFable51Added, +] as PolicyBenchNote[]; + +export function getNote(slug: string): PolicyBenchNote | undefined { + return notes.find((note) => note.slug === slug); +} diff --git a/tests/test_notes.py b/tests/test_notes.py new file mode 100644 index 0000000..89eead6 --- /dev/null +++ b/tests/test_notes.py @@ -0,0 +1,245 @@ +"""Checks that published note prose stays tied to committed evidence.""" + +from __future__ import annotations + +import csv +import gzip +import json +import re +import sys +from functools import cache +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +NOTES_DIR = ROOT / "app/src/notes" +RUN_DIR = ( + ROOT / "paper/snapshot/20260501/runs/" + "us_full_run_20260612_policyengine_4_16_1_populace" +) +DATA_PATH = RUN_DIR / "data.json" +PREDICTIONS_PATH = RUN_DIR / "predictions.csv.gz" +REFERENCES_PATH = RUN_DIR / "reference_outputs.csv" +REFERENCE_META_PATH = RUN_DIR / "reference_outputs.csv.meta.json" +PATHWAYS_PATH = ROOT / "notes/data/snap_pathways_20260901.csv" +PATHWAYS_META_PATH = PATHWAYS_PATH.with_suffix(PATHWAYS_PATH.suffix + ".meta.json") +SENSITIVITY_PATH = ROOT / "sensitivity/data/claude-fable-5-1-thinking.json" +SENSITIVITY_NOTE_PATH = ROOT / "sensitivity/claude-thinking-2026-08.md" + +CLAUDE_NOTE = "2026-09-01-claude-fable-5-1-added" +SNAP_NOTE = "2026-09-03-six-snap-households" +TOP_MODELS = ("gpt-5.6-sol", "claude-fable-5.1", "kimi-k3") +PLACEHOLDER = re.compile(r"\{([A-Za-z][A-Za-z0-9]*)\}") + +csv.field_size_limit(sys.maxsize) + + +def _load_json(path: Path) -> dict: + with path.open(encoding="utf-8") as source: + return json.load(source) + + +def _note(slug: str) -> dict: + return _load_json(NOTES_DIR / f"{slug}.json") + + +@cache +def _dashboard() -> dict: + return _load_json(DATA_PATH) + + +@cache +def _snap_predictions() -> dict[str, list[dict[str, str]]]: + rows = {model: [] for model in TOP_MODELS} + with gzip.open(PREDICTIONS_PATH, "rt", encoding="utf-8", newline="") as source: + for row in csv.DictReader(source): + if row["variable"] == "snap" and row["model"] in rows: + rows[row["model"]].append(row) + assert all(len(model_rows) == 100 for model_rows in rows.values()) + return rows + + +def _snap_references() -> dict[str, float]: + with REFERENCES_PATH.open(encoding="utf-8", newline="") as source: + return { + row["scenario_id"]: float(row["value"]) + for row in csv.DictReader(source) + if row["variable"] == "snap" + } + + +def _display_one_decimal(value: float) -> float: + return float(f"{value:.1f}") + + +def _sensitivity_row(markdown: str, label: str) -> tuple[float, float]: + match = re.search( + rf"^\|\s*{re.escape(label)}\s*\|" + r"\s*([0-9.]+)\s*\([^)]*\)\s*\|" + r"\s*(?:\*\*)?([0-9.]+)(?:\*\*)?\s*\|", + markdown, + re.MULTILINE, + ) + assert match is not None + return float(match.group(1)), float(match.group(2)) + + +@pytest.mark.parametrize("path", sorted(NOTES_DIR.glob("*.json"))) +def test_note_schema_and_placeholders(path: Path) -> None: + note = _load_json(path) + assert note["slug"] == path.stem + assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", note["date"]) + assert note["boardSnapshot"] == "2026-09-01" + assert note["release"] == "dashboard-data-20260901c" + assert note["paragraphs"] + assert note["data"] + + placeholders = { + key + for paragraph in note["paragraphs"] + for key in PLACEHOLDER.findall(paragraph) + } + assert placeholders == set(note["facts"]) + + +def test_claude_fable_note_facts() -> None: + note = _note(CLAUDE_NOTE) + board_rows = [ + row for row in _dashboard()["modelStats"] if row["condition"] == "no_tools" + ] + target = next(row for row in board_rows if row["model"] == "claude-fable-5.1") + sensitivity = _load_json(SENSITIVITY_PATH) + sensitivity_exact = float(sensitivity["sensitivity"]["exact"]) + markdown = SENSITIVITY_NOTE_PATH.read_text(encoding="utf-8") + fable5_board, fable5_auto = _sensitivity_row(markdown, "Claude Fable 5") + + derived = { + "exactRate": _display_one_decimal(target["exact"]), + "rank": 1 + sum(row["exact"] > target["exact"] for row in board_rows), + "nModels": len(board_rows), + "parsed": target["nParsed"], + "answers": target["n"], + "autoRate": _display_one_decimal(sensitivity_exact), + "autoRank": 1 + sum(row["exact"] > sensitivity_exact for row in board_rows), + "fable5AutoRate": fable5_auto, + "fable5BoardRate": fable5_board, + "boardGap": _display_one_decimal(target["exact"] - fable5_board), + } + assert note["facts"] == derived + assert sensitivity["release"] == note["release"] + + +def _reference_monthly_minimum() -> float: + entries = _dashboard()["scenarioPredictions"]["scenario_030"]["snap"].values() + explanation = next(entry["referenceExplanation"] for entry in entries) + match = re.search(r"minimum allotment of \$([0-9.]+) per month", explanation) + assert match is not None + return float(match.group(1)) + + +def _mention_count(rows: list[dict[str, str]], pattern: str) -> int: + regex = re.compile(pattern, re.IGNORECASE) + return sum(bool(regex.search(row["explanation"] or "")) for row in rows) + + +def test_six_snap_households_note_facts() -> None: + note = _note(SNAP_NOTE) + references = _snap_references() + eligible_ids = { + scenario_id for scenario_id, value in references.items() if value > 0 + } + predictions = _snap_predictions() + denied_by_model = { + model: { + row["scenario_id"] + for row in rows + if row["scenario_id"] in eligible_ids + and row["prediction"] + and float(row["prediction"]) == 0 + } + for model, rows in predictions.items() + } + denied_sets = list(denied_by_model.values()) + assert denied_sets[1:] == denied_sets[:-1] + denied_ids = sorted(denied_sets[0]) + denied_references = {references[scenario_id] for scenario_id in denied_ids} + assert len(denied_references) == 1 + + with PATHWAYS_PATH.open(encoding="utf-8", newline="") as source: + pathways = list(csv.DictReader(source)) + assert len(pathways) == 100 + assert all( + abs(float(row["snap_recomputed"]) - float(row["snap_reference"])) <= 1 + for row in pathways + ) + assert sum(row["snap_eligible"] == "True" for row in pathways) == len(eligible_ids) + categorical_rows = [ + row for row in pathways if row["pathway"].startswith("categorical_") + ] + categorical_income = [ + row + for row in pathways + if row["pathway"] in {"categorical_income", "categorical_both"} + ] + categorical_assets = [ + row for row in pathways if row["pathway"] == "categorical_assets" + ] + + pathway_meta = _load_json(PATHWAYS_META_PATH) + reference_meta = _load_json(REFERENCE_META_PATH) + regexes = note["mentionRegexes"] + derived = { + "eligibleCount": len(eligible_ids), + "deniedCount": len(denied_ids), + "deniedScenarios": denied_ids, + "referenceAnnual": round(next(iter(denied_references)), 2), + "referenceMonthly": _reference_monthly_minimum(), + "categoricalOnlyCount": len(categorical_rows), + "categoricalIncomeCount": len(categorical_income), + "categoricalAssetCount": len(categorical_assets), + "solCategoricalMentions": _mention_count( + predictions["gpt-5.6-sol"], regexes["categorical"] + ), + "solAssetMentions": _mention_count( + predictions["gpt-5.6-sol"], regexes["assets"] + ), + "fableBbceMentions": _mention_count( + predictions["claude-fable-5.1"], regexes["bbce"] + ), + "kimiCategoricalMentions": _mention_count( + predictions["kimi-k3"], regexes["categorical"] + ), + "pathwayEngineVersion": pathway_meta["policyengine_us_version"], + "referenceEngineVersion": reference_meta["policyengine_bundles"]["us"][ + "model_version" + ], + } + assert note["facts"] == derived + + +def test_categorical_asset_error_statements() -> None: + predictions = _snap_predictions() + with PATHWAYS_PATH.open(encoding="utf-8", newline="") as source: + asset_rows = { + row["scenario_id"]: float(row["snap_reference"]) + for row in csv.DictReader(source) + if row["pathway"] == "categorical_assets" + } + assert len(asset_rows) == 4 + + errors: dict[str, list[float]] = {} + for model in ("gpt-5.6-sol", "claude-fable-5.1"): + model_predictions = { + row["scenario_id"]: float(row["prediction"]) for row in predictions[model] + } + errors[model] = [ + abs(model_predictions[scenario_id] - reference) / reference + for scenario_id, reference in asset_rows.items() + ] + + sol_errors = errors["gpt-5.6-sol"] + fable_errors = errors["claude-fable-5.1"] + assert sum(error <= 0.01 for error in sol_errors) == 3 + assert sum(0.01 < error <= 0.10 for error in sol_errors) == 1 + assert sum(error <= 0.01 for error in fable_errors) == 4 From 7457d7d7c418ff64b5b1e12170b38701795b73f2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 3 Sep 2026 08:04:33 -0400 Subject: [PATCH 04/10] Add /notes: dated, data-linked board notes --- PROGRESS.md | 8 +- app/src/App.tsx | 15 ++- app/src/app/model/[id]/page.tsx | 6 +- app/src/app/notes/[slug]/page.tsx | 76 +++++++++++ app/src/app/notes/page.tsx | 63 +++++++++ app/src/app/paper/page.tsx | 9 +- app/src/app/sitemap.ts | 12 ++ app/src/components/Hero.tsx | 9 +- app/src/components/NotesContent.tsx | 193 ++++++++++++++++++++++++++++ app/src/components/SiteHeader.tsx | 25 ++-- app/tests/notes.test.ts | 44 +++++++ 11 files changed, 441 insertions(+), 19 deletions(-) create mode 100644 app/src/app/notes/[slug]/page.tsx create mode 100644 app/src/app/notes/page.tsx create mode 100644 app/src/components/NotesContent.tsx create mode 100644 app/tests/notes.test.ts diff --git a/PROGRESS.md b/PROGRESS.md index 9d1df6a..56243c2 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,7 +2,7 @@ ## State -Both notes and their frozen-evidence fact tests are implemented. +The `/notes` feature is implemented; full repository verification is underway. ## Done @@ -17,9 +17,13 @@ Both notes and their frozen-evidence fact tests are implemented. - Added Python coverage that derives every placeholder fact, the explanation mention counts, categorical pathway counts, asset-case error bands, and engine versions from committed evidence. The focused suite passes (5 tests). +- Added the `/notes` list and static detail routes, shared fact-aware rendering, + canonical metadata, navigation links, and sitemap entries. +- Added app coverage for note ordering, slugs, placeholders, and rendered + content. Lint and all 95 app tests pass; a webpack production build also + prerenders `/notes` and both detail pages successfully. ## Next -- Add the `/notes` routes, navigation, scenario deep links, sitemap entries, and app tests. - Run the requested Python and app verification suites. - Write `out.md`, push `notes-route`, and open a draft PR against `add-fable-5-1`. diff --git a/app/src/App.tsx b/app/src/App.tsx index c118533..a727362 100644 --- a/app/src/App.tsx +++ b/app/src/App.tsx @@ -1,5 +1,6 @@ "use client"; +import Link from "next/link"; import { useCallback, useEffect, @@ -353,6 +354,10 @@ export default function App() { availableViews={availableViews} navItems={navItems} activeNav={activeNav} + actionLinks={[ + { label: "Paper", href: "/paper", type: "internal" }, + { label: "Notes", href: "/notes", type: "internal" }, + ]} versionId={versionId} pendingVersionId={pendingVersionId} onSelectVersion={handleSelectVersion} @@ -442,6 +447,13 @@ export default function App() { Paper {" "} ·{" "} + + Notes + {" "} + ·{" "} Expand - + {" "} + ·{" "} diff --git a/app/src/app/notes/[slug]/page.tsx b/app/src/app/notes/[slug]/page.tsx new file mode 100644 index 0000000..f541814 --- /dev/null +++ b/app/src/app/notes/[slug]/page.tsx @@ -0,0 +1,76 @@ +import type { Metadata } from "next"; +import { notFound } from "next/navigation"; + +import { NoteArticle, interpolateNoteText } from "../../../components/NotesContent"; +import SiteHeader from "../../../components/SiteHeader"; +import { getNote, notes } from "../../../notes"; + +export const dynamicParams = false; + +export function generateStaticParams() { + return notes.map((note) => ({ slug: note.slug })); +} + +export async function generateMetadata({ + params, +}: { + params: Promise<{ slug: string }>; +}): Promise { + const { slug } = await params; + const note = getNote(slug); + if (!note) notFound(); + const description = interpolateNoteText(note, note.paragraphs[0]); + const url = `https://policybench.org/notes/${note.slug}`; + return { + title: note.title, + description, + alternates: { canonical: `/notes/${note.slug}` }, + openGraph: { + type: "article", + url, + siteName: "PolicyBench", + title: note.title, + description, + publishedTime: note.date, + images: [ + { + url: "/og-image.png", + width: 1200, + height: 630, + alt: "PolicyBench β€” an LLM benchmark for tax and benefit calculation", + }, + ], + }, + twitter: { + card: "summary_large_image", + title: note.title, + description, + images: ["/og-image.png"], + }, + }; +} + +export default async function NotePage({ + params, +}: { + params: Promise<{ slug: string }>; +}) { + const { slug } = await params; + const note = getNote(slug); + if (!note) notFound(); + + return ( +
+ +
+ +
+
+ ); +} diff --git a/app/src/app/notes/page.tsx b/app/src/app/notes/page.tsx new file mode 100644 index 0000000..d72516a --- /dev/null +++ b/app/src/app/notes/page.tsx @@ -0,0 +1,63 @@ +import type { Metadata } from "next"; + +import { + NOTES_INTRO, + NotesPageContent, +} from "../../components/NotesContent"; +import SiteHeader from "../../components/SiteHeader"; + +export const metadata: Metadata = { + title: "Notes", + description: NOTES_INTRO, + alternates: { + canonical: "/notes", + }, + openGraph: { + type: "website", + url: "https://policybench.org/notes", + siteName: "PolicyBench", + title: "PolicyBench notes", + description: NOTES_INTRO, + images: [ + { + url: "/og-image.png", + width: 1200, + height: 630, + alt: "PolicyBench β€” an LLM benchmark for tax and benefit calculation", + }, + ], + }, + twitter: { + card: "summary_large_image", + title: "PolicyBench notes", + description: NOTES_INTRO, + images: ["/og-image.png"], + }, +}; + +export default function NotesPage() { + const expanded = ( + <> +

+ Notes +

+

+ {NOTES_INTRO} +

+ + ); + + return ( +
+ + +
+ ); +} diff --git a/app/src/app/paper/page.tsx b/app/src/app/paper/page.tsx index 550994c..9cc6b5a 100644 --- a/app/src/app/paper/page.tsx +++ b/app/src/app/paper/page.tsx @@ -69,11 +69,10 @@ export default function PaperPage() {

PolicyBench paper

diff --git a/app/src/app/sitemap.ts b/app/src/app/sitemap.ts index 816f7cc..6d26dbf 100644 --- a/app/src/app/sitemap.ts +++ b/app/src/app/sitemap.ts @@ -2,6 +2,7 @@ import type { MetadataRoute } from "next"; import rawData from "../data-summary.json"; import { listModels } from "../lib/modelPage"; +import { notes } from "../notes"; import type { DashboardBundle } from "../types"; export default function sitemap(): MetadataRoute.Sitemap { @@ -10,6 +11,11 @@ export default function sitemap(): MetadataRoute.Sitemap { changeFrequency: "monthly" as const, priority: 0.6, })); + const noteEntries = notes.map((note) => ({ + url: `https://policybench.org/notes/${note.slug}`, + changeFrequency: "monthly" as const, + priority: 0.5, + })); return [ { url: "https://policybench.org/", @@ -21,11 +27,17 @@ export default function sitemap(): MetadataRoute.Sitemap { changeFrequency: "monthly", priority: 0.7, }, + { + url: "https://policybench.org/notes", + changeFrequency: "weekly", + priority: 0.7, + }, { url: "https://policybench.org/expand", changeFrequency: "monthly", priority: 0.5, }, + ...noteEntries, ...modelEntries, ]; } diff --git a/app/src/components/Hero.tsx b/app/src/components/Hero.tsx index 48a64cc..8387420 100644 --- a/app/src/components/Hero.tsx +++ b/app/src/components/Hero.tsx @@ -1,6 +1,9 @@ import { DEFAULT_VERSION_ID } from "../lib/dataVersionsRuntime"; import type { BenchData, CountryCode } from "../types"; -import SiteHeader, { type HeaderNavItem } from "./SiteHeader"; +import SiteHeader, { + type HeaderActionLink, + type HeaderNavItem, +} from "./SiteHeader"; export default function Hero({ selectedView, @@ -9,6 +12,7 @@ export default function Hero({ availableViews, navItems, activeNav, + actionLinks, versionId, pendingVersionId, onSelectVersion, @@ -20,6 +24,7 @@ export default function Hero({ availableViews: CountryCode[]; navItems: readonly HeaderNavItem[]; activeNav: string; + actionLinks: readonly HeaderActionLink[]; versionId: string; pendingVersionId: string | null; onSelectVersion: (id: string) => void; @@ -56,7 +61,7 @@ export default function Hero({ versionId={versionId} pendingVersionId={pendingVersionId} onSelectVersion={onSelectVersion} - actionLink={{ label: "Paper", href: "/paper", type: "internal" }} + actionLinks={actionLinks} />
{ + const fact = note.facts[key]; + if (fact === undefined) throw new Error(`Unknown note fact: ${key}`); + return factText(key, fact); + }); +} + +function NoteLink({ href, children }: { href: string; children: ReactNode }) { + const className = "text-primary-strong underline-offset-2 hover:underline"; + if (href.startsWith("/")) { + return ( + + {children} + + ); + } + return ( + + {children} + + ); +} + +function factNode( + note: PolicyBenchNote, + key: string, + fact: NoteFact, +): ReactNode { + if (Array.isArray(fact)) { + return fact.map((item, index) => { + const link = note.data.find((entry) => entry.label === item); + return ( + + {index > 0 ? ", " : null} + {link ? {item} : item} + + ); + }); + } + if (key === "referenceAnnual" && typeof fact === "number") { + return ( + <> + {Math.round(fact)} + + + 1 + + + + ); + } + return String(fact); +} + +function NoteParagraph({ + note, + paragraph, +}: { + note: PolicyBenchNote; + paragraph: string; +}) { + const content: ReactNode[] = []; + let cursor = 0; + for (const match of paragraph.matchAll(PLACEHOLDER)) { + const index = match.index; + if (index > cursor) content.push(paragraph.slice(cursor, index)); + const key = match[1]; + const fact = note.facts[key]; + if (fact === undefined) throw new Error(`Unknown note fact: ${key}`); + content.push({factNode(note, key, fact)}); + cursor = index + match[0].length; + } + if (cursor < paragraph.length) content.push(paragraph.slice(cursor)); + return

{content}

; +} + +export function NoteArticle({ + note, + titleLevel, + linkTitle = false, +}: { + note: PolicyBenchNote; + titleLevel: "h1" | "h2"; + linkTitle?: boolean; +}) { + const Title = titleLevel; + const title = linkTitle ? ( + + {note.title} + + ) : ( + note.title + ); + const annualReference = note.facts.referenceAnnual; + const DataTitle = titleLevel === "h1" ? "h2" : "h3"; + + return ( +
+ + + {title} + + +
+ {note.paragraphs.map((paragraph, index) => ( + + ))} +
+ + {typeof annualReference === "number" ? ( +

+ 1 Unrounded frozen reference: ${annualReference.toFixed(2)}. +

+ ) : null} + +
+ + Data + +
    + {note.data.map((entry) => ( +
  • + {entry.label} +
    • + ))} +
+
+
+ ); +} + +export function NotesPageContent() { + return ( +
+
+ {notes.map((note, index) => ( + + {index > 0 ? ( +
+ ) : null} + + + ))} +
+
+ ); +} diff --git a/app/src/components/SiteHeader.tsx b/app/src/components/SiteHeader.tsx index fe69491..00b4c43 100644 --- a/app/src/components/SiteHeader.tsx +++ b/app/src/components/SiteHeader.tsx @@ -151,6 +151,7 @@ export type SiteHeaderProps = { onSelectView?: (view: CountryCode) => void; availableViews?: CountryCode[]; actionLink?: HeaderActionLink; + actionLinks?: readonly HeaderActionLink[]; /** Active dataset-version id; when set with `onSelectVersion`, shows the * dataset selector in the header. */ versionId?: string; @@ -179,6 +180,7 @@ export default function SiteHeader({ onSelectView, availableViews, actionLink, + actionLinks, versionId, pendingVersionId, onSelectVersion, @@ -209,6 +211,10 @@ export default function SiteHeader({ availableViews && availableViews.length > 0 && selectedView && onSelectView; const showVersionSelector = versionId != null && onSelectVersion != null; const headerPositionClass = alwaysExpanded ? "relative z-40" : "sticky top-0 z-40"; + const resolvedActionLinks = [ + ...(actionLinks ?? []), + ...(actionLink ? [actionLink] : []), + ]; return (
@@ -304,25 +310,28 @@ export default function SiteHeader({
)} - {actionLink && ( -
- {actionLink.type === "external" ? ( + {resolvedActionLinks.map((link) => ( +
+ {link.type === "external" ? ( - {actionLink.label} + {link.label} ) : ( - {actionLink.label} + {link.label} )}
- )} + ))} { + test("the index is newest first", () => { + expect(notes.map((note) => note.date)).toEqual( + [...notes].map((note) => note.date).sort().reverse(), + ); + }); + + test("slugs are unique and match JSON filenames", () => { + const slugs = notes.map((note) => note.slug); + expect(new Set(slugs).size).toBe(slugs.length); + const filenames = readdirSync(new URL("../src/notes", import.meta.url)) + .filter((name) => name.endsWith(".json")) + .map((name) => name.replace(/\.json$/, "")) + .sort(); + expect([...slugs].sort()).toEqual(filenames); + }); + + test("every paragraph placeholder resolves", () => { + for (const note of notes) { + const resolved = note.paragraphs.map((paragraph) => + interpolateNoteText(note, paragraph), + ); + expect(resolved.join(" ")).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*\}/); + } + }); + + test("the notes page renders both titles without unresolved placeholders", () => { + const markup = renderToStaticMarkup(createElement(NotesPageContent)); + expect(markup).toContain("Six SNAP households the top three models deny"); + expect(markup).toContain("Claude Fable 5.1 added"); + expect(markup).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*\}/); + }); +}); From 9d1374dac80e000521fbf14a719438008b955cc1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 3 Sep 2026 08:06:47 -0400 Subject: [PATCH 05/10] Fix PolicyEngine parameter links --- app/src/notes/2026-09-03-six-snap-households.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/notes/2026-09-03-six-snap-households.json b/app/src/notes/2026-09-03-six-snap-households.json index 8f983cf..b97ddbb 100644 --- a/app/src/notes/2026-09-03-six-snap-households.json +++ b/app/src/notes/2026-09-03-six-snap-households.json @@ -88,11 +88,11 @@ }, { "label": "TANF non-cash parameters", - "href": "https://github.com/PolicyEngine/policyengine-us/tree/master/policyengine_us/parameters/gov/hhs/tanf/non_cash" + "href": "https://github.com/PolicyEngine/policyengine-us/tree/main/policyengine_us/parameters/gov/hhs/tanf/non_cash" }, { "label": "SNAP minimum-allotment parameters", - "href": "https://github.com/PolicyEngine/policyengine-us/tree/master/policyengine_us/parameters/gov/usda/snap/min_allotment" + "href": "https://github.com/PolicyEngine/policyengine-us/tree/main/policyengine_us/parameters/gov/usda/snap/min_allotment" } ], "boardSnapshot": "2026-09-01", From 0fd160cff7e60ee0f8b5cce0739e0a53443de3ac Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 3 Sep 2026 08:14:56 -0400 Subject: [PATCH 06/10] Document notes route delivery --- PROGRESS.md | 12 ++- out.md | 215 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+), 3 deletions(-) create mode 100644 out.md diff --git a/PROGRESS.md b/PROGRESS.md index 56243c2..8bcfebd 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,7 +2,8 @@ ## State -The `/notes` feature is implemented; full repository verification is underway. +The `/notes` feature and its delivery report are complete and verified. The +branch is ready to push and open as a draft pull request. ## Done @@ -22,8 +23,13 @@ The `/notes` feature is implemented; full repository verification is underway. - Added app coverage for note ordering, slugs, placeholders, and rendered content. Lint and all 95 app tests pass; a webpack production build also prerenders `/notes` and both detail pages successfully. +- Ran the full Python gate: Ruff passes and pytest reports 623 passed, 5 + skipped. Re-ran the exact app lint and test commands from CI successfully. +- Wrote the required delivery report to `out.md` with derived facts, pathway + results, exact test output, and the local build limitation. ## Next -- Run the requested Python and app verification suites. -- Write `out.md`, push `notes-route`, and open a draft PR against `add-fable-5-1`. +- Push `notes-route` and open a draft PR against `add-fable-5-1`. +- Add the draft PR URL to `out.md`, mark delivery complete, and push that final + report commit. diff --git a/out.md b/out.md new file mode 100644 index 0000000..e9925e6 --- /dev/null +++ b/out.md @@ -0,0 +1,215 @@ +# `/notes` delivery report + +## State + +Implemented and verified on `notes-route`. The draft PR URL is pending until +the branch is pushed; it will be inserted in a final committed update. + +## Notes and routes + +- Added `/notes`, newest first, and statically generated detail routes: + - `/notes/2026-09-03-six-snap-households` + - `/notes/2026-09-01-claude-fable-5-1-added` +- Added the two JSON note records and their typed index under + `app/src/notes/`. +- Rendered every prose placeholder from its tested `facts` entry. Scenario IDs + in the SNAP note link directly to the existing scenario explorer query + convention. The annual `$287.68` fact renders as `$288` with an exact-value + footnote. +- Added full metadata, canonical URLs, site navigation, and sitemap entries. +- Added every requested release, snapshot, sensitivity, model, paper, pathway, + scenario, and policy-parameter data link. + +## Checked note facts + +### Claude Fable 5.1 added + +- Board exact rate: 86.3%; rank: 2 of 33; parsed: 1,984 of 1,984. +- Auto-tool sensitivity: 87.5%; would-rank: 2. +- Claude Fable 5 auto rate: 86.9%; board rate: 79.9%; board-row gap: + 6.4 points. + +### Six SNAP households the top three models deny + +- Positive frozen SNAP references: 20 of 100. +- Common zero predictions from GPT-5.6 Sol, Claude Fable 5.1, and Kimi K3: + `scenario_027`, `scenario_030`, `scenario_045`, `scenario_073`, + `scenario_108`, and `scenario_112`. +- Each common-denial reference is `$287.68` annually; the frozen explanation + records `$23.84` monthly. +- Explanation-row regex counts, case-insensitive: + - Sol `categorical`: 1. + - Sol `asset|resource`: 36. + - Fable 5.1 `broad-based|bbce`: 28. + - Kimi K3 `categorical`: 17. +- Reference engine: policyengine-us 1.755.4. Pathway recompute engine: + policyengine-us 1.723.0. + +## Pathway recompute + +`scripts/snap_pathways.py` reuses the vectorized situation builder and value +extractor from `policybench/ground_truth.py`. It generated 100 committed rows +plus a provenance sidecar. + +- `ordinary`: 10 +- `categorical_income`: 5 +- `categorical_assets`: 4 +- `categorical_both`: 1 +- `ineligible`: 80 +- Positive-allotment `snap_eligible`: 20 +- Categorical-only total: 10; income-related categorical total (including + `categorical_both`): 6. +- Asset-only cases and states: + - `scenario_008` β€” New Jersey + - `scenario_054` β€” North Carolina + - `scenario_066` β€” Virginia + - `scenario_080` β€” Pennsylvania +- On those four asset cases, Sol is within 1% on three and within 10% on one; + Fable 5.1 is within 1% on all four. +- Recompute/reference disagreements: none. Maximum absolute difference: `$0`. +- The underlying `is_snap_eligible` gate is true despite a zero computed + allotment for `scenario_028` and `scenario_032`; the exported eligibility + column deliberately follows the note's positive-allotment definition. + +One-time script output: + +```text +PolicyEngine's is_snap_eligible gate is true but the computed allotment is $0 for: scenario_028, scenario_032 +Wrote 100 rows to notes/data/snap_pathways_20260901.csv + ordinary: 10 + categorical_income: 5 + categorical_assets: 4 + categorical_both: 1 + ineligible: 80 +``` + +## Files changed + +- Progress and report: `PROGRESS.md`, `out.md`. +- Evidence generation: `scripts/snap_pathways.py`, + `notes/data/snap_pathways_20260901.csv`, and its `.meta.json` sidecar. +- Note records and rendering: `app/src/notes/`, + `app/src/components/NotesContent.tsx`, and the two `/notes` page files. +- Discovery and navigation: `app/src/App.tsx`, `app/src/components/Hero.tsx`, + `app/src/components/SiteHeader.tsx`, the paper and model pages, and the + sitemap. +- Tests: `tests/test_notes.py` and `app/tests/notes.test.ts`. +- Ignore rule: `.gitignore` admits the committed pathway evidence. + +No file under `paper/snapshot/**` or `app/public/paper/**` was edited. + +## Verification + +### Python + +Command: + +```text +PY=/Users/maxghenis/PolicyEngine/policybench/.venv/bin/python +PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/notes +$PY -c '' +$PY -m ruff format . +$PY -m ruff check . +$PY -m pytest -q +``` + +Output: + +```text +policybench.ground_truth: /Users/maxghenis/PolicyEngine/policybench-wt/notes/policybench/ground_truth.py +75 files left unchanged +All checks passed! +........................................................................ [ 11%] +........................................................................ [ 22%] +........................................................................ [ 34%] +........................................................................ [ 45%] +........................................................................ [ 57%] +........................................................................ [ 68%] +........................................................................ [ 80%] +...sss...ss............................................................. [ 91%] +.................................................... [100%] +=============================== warnings summary =============================== +tests/test_ground_truth.py::TestGroundTruth::test_eitc_zero_for_50k_single +tests/test_ground_truth.py::TestGroundTruth::test_eitc_positive_for_low_income_family +tests/test_ground_truth.py::TestGroundTruth::test_income_tax_reconciles_from_compact_tax_components +tests/test_ground_truth.py::TestGroundTruth::test_household_structure_drives_filing_status_ground_truth +tests/test_ground_truth.py::TestGroundTruth::test_household_net_income_reasonable +tests/test_ground_truth.py::TestGroundTruth::test_calculate_ground_truth_dataframe +tests/test_ground_truth.py::TestGroundTruth::test_us_vectorized_ground_truth_matches_scalar_reference +tests/test_ground_truth.py::TestGroundTruth::test_us_vectorized_ground_truth_matches_scalar_reference + /Users/maxghenis/PolicyEngine/policybench/.venv/lib/python3.14/site-packages/policyengine_us/variables/gov/irs/income/taxable_income/adjusted_gross_income/irs_gross_income/unemployment_insurance/taxable_unemployment_insurance.py:22: RuntimeWarning: invalid value encountered in divide + share = where(tax_unit_uc > 0, person_uc / tax_unit_uc, 0) + +tests/test_snapshot_artifacts.py::test_snapshot_deviation_audit_annotations_are_complete_and_final +tests/test_snapshot_artifacts.py::test_snapshot_audit_annotations_have_no_orphan_rows + /Users/maxghenis/PolicyEngine/policybench-wt/notes/policybench/annotation_validation.py:226: DtypeWarning: Columns (0: cost_is_estimated) have mixed types. Specify dtype option on import or set low_memory=False. + predictions = pd.read_csv(country_dir / "predictions.csv.gz") + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +623 passed, 5 skipped, 10 warnings in 50.54s +``` + +### App lint and tests + +Command, matching the app CI job: + +```text +bun run lint +bun test tests +``` + +Output: + +```text +$ eslint . --max-warnings=0 +bun test v1.3.12 (700fc117) + + 95 pass + 0 fail + 322 expect() calls +Ran 95 tests across 12 files. [1316.00ms] +``` + +### Production build + +`IS_WEBPACK_TEST=1 bun run build` completed successfully with Next.js 16.2.6 +using webpack. It compiled, type-checked, generated all 43 static pages, and +reported: + +```text +Route (app) +β”Œ β—‹ / +β”œ β—‹ /_not-found +β”œ β—‹ /expand +β”œ ● /model/[id] +β”‚ β”œ /model/gpt-5.6-sol +β”‚ β”œ /model/claude-fable-5.1 +β”‚ β”œ /model/kimi-k3 +β”‚ β”” [+30 more paths] +β”œ β—‹ /notes +β”œ ● /notes/[slug] +β”‚ β”œ /notes/2026-09-03-six-snap-households +β”‚ β”” /notes/2026-09-01-claude-fable-5-1-added +β”œ β—‹ /paper +β”œ β—‹ /robots.txt +β”” β—‹ /sitemap.xml + +β—‹ (Static) prerendered as static content +● (SSG) prerendered as static HTML (uses generateStaticParams) +``` + +## Delivery + +- Branch: `notes-route` +- Base: `add-fable-5-1` +- Draft PR: pending branch push + +## Could not do locally + +The default Turbopack path behind unprefixed `bun run build` fails in this +managed sandbox with `TurbopackInternalError: Failed to write app endpoint +/page`. Its cause chain ends in PostCSS loader evaluation trying to create a +process, bind a port, and receiving `Operation not permitted (os error 1)`. +`NEXT_TURBOPACK_USE_WORKER=0` reaches the same sandbox restriction. The webpack +build path completed successfully as recorded above. CI runs outside this +sandbox and can execute the default command. From e2cdd895c18f321e910fc48838251c1a37d27a05 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 3 Sep 2026 08:16:49 -0400 Subject: [PATCH 07/10] Record blocked GitHub delivery --- PROGRESS.md | 14 +++++++++----- out.md | 28 +++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 8bcfebd..fd81895 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,8 +2,9 @@ ## State -The `/notes` feature and its delivery report are complete and verified. The -branch is ready to push and open as a draft pull request. +The `/notes` feature and its delivery report are complete and verified. Remote +delivery is blocked by this workspace's GitHub connectivity and connector +authorization; no remote branch or pull request was created. ## Done @@ -27,9 +28,12 @@ branch is ready to push and open as a draft pull request. skipped. Re-ran the exact app lint and test commands from CI successfully. - Wrote the required delivery report to `out.md` with derived facts, pathway results, exact test output, and the local build limitation. +- Attempted the requested push. Git could not resolve `github.com`; the GitHub + connector fallback returned `user cancelled MCP tool call` for both tree and + branch creation, leaving the remote unchanged. ## Next -- Push `notes-route` and open a draft PR against `add-fable-5-1`. -- Add the draft PR URL to `out.md`, mark delivery complete, and push that final - report commit. +- Once GitHub access is restored, push `notes-route`. +- Open a draft PR against `add-fable-5-1` and add its URL to `out.md` in a + final committed update. diff --git a/out.md b/out.md index e9925e6..0a21f83 100644 --- a/out.md +++ b/out.md @@ -2,8 +2,9 @@ ## State -Implemented and verified on `notes-route`. The draft PR URL is pending until -the branch is pushed; it will be inserted in a final committed update. +Implemented and verified on the local `notes-route` branch. Remote delivery is +blocked by this workspace's GitHub connectivity and connector authorization; +details and exact handoff commands are below. ## Notes and routes @@ -202,7 +203,15 @@ Route (app) - Branch: `notes-route` - Base: `add-fable-5-1` -- Draft PR: pending branch push +- Local implementation commit before this report update: `725c1ac` +- Draft PR URL: unavailable because the remote branch could not be created. + +The remaining commands, once GitHub access is restored, are: + +```text +git push -u origin notes-route +gh pr create --draft --base add-fable-5-1 --head notes-route --title "Add /notes with two data-linked notes" --body "Adds dated, fact-tested notes for Claude Fable 5.1 and six shared SNAP denials, with committed pathway evidence." +``` ## Could not do locally @@ -213,3 +222,16 @@ process, bind a port, and receiving `Operation not permitted (os error 1)`. `NEXT_TURBOPACK_USE_WORKER=0` reaches the same sandbox restriction. The webpack build path completed successfully as recorded above. CI runs outside this sandbox and can execute the default command. + +The requested push failed before authentication because the managed shell +cannot resolve GitHub: + +```text +fatal: unable to access 'https://github.com/PolicyEngine/policybench.git/': Could not resolve host: github.com +``` + +The installed GitHub connector was then tried as the network-independent +fallback. Both low-level tree creation and direct `notes-route` branch creation +returned `user cancelled MCP tool call`; a follow-up branch search confirmed +that `notes-route` does not exist remotely. Therefore no truthful PR URL can be +recorded and no draft PR can be opened from this environment. From 7d72ca4dac26d5dcca90922a1a2ce428674cc7b2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 3 Sep 2026 08:17:54 -0400 Subject: [PATCH 08/10] Tighten note wording, format thousands, drop lane files --- PROGRESS.md | 39 --- app/src/components/NotesContent.tsx | 5 +- .../notes/2026-09-03-six-snap-households.json | 4 +- out.md | 237 ------------------ 4 files changed, 6 insertions(+), 279 deletions(-) delete mode 100644 PROGRESS.md delete mode 100644 out.md diff --git a/PROGRESS.md b/PROGRESS.md deleted file mode 100644 index fd81895..0000000 --- a/PROGRESS.md +++ /dev/null @@ -1,39 +0,0 @@ -# Progress - -## State - -The `/notes` feature and its delivery report are complete and verified. Remote -delivery is blocked by this workspace's GitHub connectivity and connector -authorization; no remote branch or pull request was created. - -## Done - -- Installed the app dependencies with `bun install --frozen-lockfile`. -- Confirmed the worktree is on `notes-route` at the requested starting commit. -- Added `scripts/snap_pathways.py`, reusing the reference calculation's - vectorized situation builder and the installed PolicyEngine variables. -- Generated 100 pathway rows with policyengine-us 1.723.0. All recomputed SNAP - values match the frozen references exactly; 20 households receive SNAP and - 10 depend on categorical eligibility. -- Added the two dated note JSON files and a newest-first TypeScript index. -- Added Python coverage that derives every placeholder fact, the explanation - mention counts, categorical pathway counts, asset-case error bands, and - engine versions from committed evidence. The focused suite passes (5 tests). -- Added the `/notes` list and static detail routes, shared fact-aware rendering, - canonical metadata, navigation links, and sitemap entries. -- Added app coverage for note ordering, slugs, placeholders, and rendered - content. Lint and all 95 app tests pass; a webpack production build also - prerenders `/notes` and both detail pages successfully. -- Ran the full Python gate: Ruff passes and pytest reports 623 passed, 5 - skipped. Re-ran the exact app lint and test commands from CI successfully. -- Wrote the required delivery report to `out.md` with derived facts, pathway - results, exact test output, and the local build limitation. -- Attempted the requested push. Git could not resolve `github.com`; the GitHub - connector fallback returned `user cancelled MCP tool call` for both tree and - branch creation, leaving the remote unchanged. - -## Next - -- Once GitHub access is restored, push `notes-route`. -- Open a draft PR against `add-fable-5-1` and add its URL to `out.md` in a - final committed update. diff --git a/app/src/components/NotesContent.tsx b/app/src/components/NotesContent.tsx index fa25a53..e690d2f 100644 --- a/app/src/components/NotesContent.tsx +++ b/app/src/components/NotesContent.tsx @@ -22,6 +22,9 @@ function factText(key: string, fact: NoteFact): string { if (key === "referenceAnnual" && typeof fact === "number") { return String(Math.round(fact)); } + if (typeof fact === "number" && Number.isInteger(fact)) { + return fact.toLocaleString("en-US"); + } return String(fact); } @@ -84,7 +87,7 @@ function factNode( ); } - return String(fact); + return factText(key, fact); } function NoteParagraph({ diff --git a/app/src/notes/2026-09-03-six-snap-households.json b/app/src/notes/2026-09-03-six-snap-households.json index b97ddbb..0e4d599 100644 --- a/app/src/notes/2026-09-03-six-snap-households.json +++ b/app/src/notes/2026-09-03-six-snap-households.json @@ -5,8 +5,8 @@ "paragraphs": [ "{eligibleCount} of the 100 households qualify for SNAP in the PolicyEngine reference. The top three models on the board, GPT-5.6 Sol, Claude Fable 5.1, and Kimi K3, each predict $0 for the same {deniedCount}: {deniedScenarios}. The reference for each is ${referenceAnnual} for the year, the minimum allotment (8% of the one-person maximum, ${referenceMonthly} a month, for households of one or two).", "Five of the six qualify through broad-based categorical eligibility: Connecticut, Michigan, Texas, and Wisconsin confer SNAP eligibility on households receiving a TANF-funded non-cash benefit, with gross-income limits of 200% of poverty (CT, MI, WI) and 165% (TX) and the net-income and asset tests waived. The sixth, scenario_112 in Texas, passes the federal tests; the models counted farm-rent income the rules exclude.", - "{categoricalOnlyCount} of the {eligibleCount} eligible households qualify only through categorical eligibility: {categoricalIncomeCount} because income exceeds the federal limits, {categoricalAssetCount} because savings exceed the federal asset limit. On the {categoricalAssetCount} asset cases the models compute benefits near the reference: Sol within 1% on three and 10% on one, Fable 5.1 within 1% on all four.", - "In their SNAP explanations across the 100 households, Sol mentions categorical eligibility {solCategoricalMentions} time(s) and assets {solAssetMentions} times; Fable 5.1 mentions broad-based categorical eligibility {fableBbceMentions} times; Kimi K3 mentions categorical eligibility {kimiCategoricalMentions} times.", + "{categoricalOnlyCount} of the {eligibleCount} eligible households qualify only through categorical eligibility: {categoricalIncomeCount} because income exceeds the federal limits (one of those also exceeds the asset limit), {categoricalAssetCount} because savings alone exceed the federal asset limit. On the {categoricalAssetCount} asset cases the models compute benefits near the reference: Sol within 1% on three and 10% on one, Fable 5.1 within 1% on all four.", + "Across the 100 households, Sol's SNAP explanations mention categorical eligibility in {solCategoricalMentions} and assets in {solAssetMentions}; Fable 5.1's mention broad-based categorical eligibility in {fableBbceMentions}; Kimi K3's mention categorical eligibility in {kimiCategoricalMentions}.", "Pathways were recomputed with policyengine-us {pathwayEngineVersion} from the frozen scenarios and agree with the frozen reference SNAP values for all 100 households; the reference itself was generated with policyengine-us {referenceEngineVersion}." ], "facts": { diff --git a/out.md b/out.md deleted file mode 100644 index 0a21f83..0000000 --- a/out.md +++ /dev/null @@ -1,237 +0,0 @@ -# `/notes` delivery report - -## State - -Implemented and verified on the local `notes-route` branch. Remote delivery is -blocked by this workspace's GitHub connectivity and connector authorization; -details and exact handoff commands are below. - -## Notes and routes - -- Added `/notes`, newest first, and statically generated detail routes: - - `/notes/2026-09-03-six-snap-households` - - `/notes/2026-09-01-claude-fable-5-1-added` -- Added the two JSON note records and their typed index under - `app/src/notes/`. -- Rendered every prose placeholder from its tested `facts` entry. Scenario IDs - in the SNAP note link directly to the existing scenario explorer query - convention. The annual `$287.68` fact renders as `$288` with an exact-value - footnote. -- Added full metadata, canonical URLs, site navigation, and sitemap entries. -- Added every requested release, snapshot, sensitivity, model, paper, pathway, - scenario, and policy-parameter data link. - -## Checked note facts - -### Claude Fable 5.1 added - -- Board exact rate: 86.3%; rank: 2 of 33; parsed: 1,984 of 1,984. -- Auto-tool sensitivity: 87.5%; would-rank: 2. -- Claude Fable 5 auto rate: 86.9%; board rate: 79.9%; board-row gap: - 6.4 points. - -### Six SNAP households the top three models deny - -- Positive frozen SNAP references: 20 of 100. -- Common zero predictions from GPT-5.6 Sol, Claude Fable 5.1, and Kimi K3: - `scenario_027`, `scenario_030`, `scenario_045`, `scenario_073`, - `scenario_108`, and `scenario_112`. -- Each common-denial reference is `$287.68` annually; the frozen explanation - records `$23.84` monthly. -- Explanation-row regex counts, case-insensitive: - - Sol `categorical`: 1. - - Sol `asset|resource`: 36. - - Fable 5.1 `broad-based|bbce`: 28. - - Kimi K3 `categorical`: 17. -- Reference engine: policyengine-us 1.755.4. Pathway recompute engine: - policyengine-us 1.723.0. - -## Pathway recompute - -`scripts/snap_pathways.py` reuses the vectorized situation builder and value -extractor from `policybench/ground_truth.py`. It generated 100 committed rows -plus a provenance sidecar. - -- `ordinary`: 10 -- `categorical_income`: 5 -- `categorical_assets`: 4 -- `categorical_both`: 1 -- `ineligible`: 80 -- Positive-allotment `snap_eligible`: 20 -- Categorical-only total: 10; income-related categorical total (including - `categorical_both`): 6. -- Asset-only cases and states: - - `scenario_008` β€” New Jersey - - `scenario_054` β€” North Carolina - - `scenario_066` β€” Virginia - - `scenario_080` β€” Pennsylvania -- On those four asset cases, Sol is within 1% on three and within 10% on one; - Fable 5.1 is within 1% on all four. -- Recompute/reference disagreements: none. Maximum absolute difference: `$0`. -- The underlying `is_snap_eligible` gate is true despite a zero computed - allotment for `scenario_028` and `scenario_032`; the exported eligibility - column deliberately follows the note's positive-allotment definition. - -One-time script output: - -```text -PolicyEngine's is_snap_eligible gate is true but the computed allotment is $0 for: scenario_028, scenario_032 -Wrote 100 rows to notes/data/snap_pathways_20260901.csv - ordinary: 10 - categorical_income: 5 - categorical_assets: 4 - categorical_both: 1 - ineligible: 80 -``` - -## Files changed - -- Progress and report: `PROGRESS.md`, `out.md`. -- Evidence generation: `scripts/snap_pathways.py`, - `notes/data/snap_pathways_20260901.csv`, and its `.meta.json` sidecar. -- Note records and rendering: `app/src/notes/`, - `app/src/components/NotesContent.tsx`, and the two `/notes` page files. -- Discovery and navigation: `app/src/App.tsx`, `app/src/components/Hero.tsx`, - `app/src/components/SiteHeader.tsx`, the paper and model pages, and the - sitemap. -- Tests: `tests/test_notes.py` and `app/tests/notes.test.ts`. -- Ignore rule: `.gitignore` admits the committed pathway evidence. - -No file under `paper/snapshot/**` or `app/public/paper/**` was edited. - -## Verification - -### Python - -Command: - -```text -PY=/Users/maxghenis/PolicyEngine/policybench/.venv/bin/python -PYTHONPATH=/Users/maxghenis/PolicyEngine/policybench-wt/notes -$PY -c '' -$PY -m ruff format . -$PY -m ruff check . -$PY -m pytest -q -``` - -Output: - -```text -policybench.ground_truth: /Users/maxghenis/PolicyEngine/policybench-wt/notes/policybench/ground_truth.py -75 files left unchanged -All checks passed! -........................................................................ [ 11%] -........................................................................ [ 22%] -........................................................................ [ 34%] -........................................................................ [ 45%] -........................................................................ [ 57%] -........................................................................ [ 68%] -........................................................................ [ 80%] -...sss...ss............................................................. [ 91%] -.................................................... [100%] -=============================== warnings summary =============================== -tests/test_ground_truth.py::TestGroundTruth::test_eitc_zero_for_50k_single -tests/test_ground_truth.py::TestGroundTruth::test_eitc_positive_for_low_income_family -tests/test_ground_truth.py::TestGroundTruth::test_income_tax_reconciles_from_compact_tax_components -tests/test_ground_truth.py::TestGroundTruth::test_household_structure_drives_filing_status_ground_truth -tests/test_ground_truth.py::TestGroundTruth::test_household_net_income_reasonable -tests/test_ground_truth.py::TestGroundTruth::test_calculate_ground_truth_dataframe -tests/test_ground_truth.py::TestGroundTruth::test_us_vectorized_ground_truth_matches_scalar_reference -tests/test_ground_truth.py::TestGroundTruth::test_us_vectorized_ground_truth_matches_scalar_reference - /Users/maxghenis/PolicyEngine/policybench/.venv/lib/python3.14/site-packages/policyengine_us/variables/gov/irs/income/taxable_income/adjusted_gross_income/irs_gross_income/unemployment_insurance/taxable_unemployment_insurance.py:22: RuntimeWarning: invalid value encountered in divide - share = where(tax_unit_uc > 0, person_uc / tax_unit_uc, 0) - -tests/test_snapshot_artifacts.py::test_snapshot_deviation_audit_annotations_are_complete_and_final -tests/test_snapshot_artifacts.py::test_snapshot_audit_annotations_have_no_orphan_rows - /Users/maxghenis/PolicyEngine/policybench-wt/notes/policybench/annotation_validation.py:226: DtypeWarning: Columns (0: cost_is_estimated) have mixed types. Specify dtype option on import or set low_memory=False. - predictions = pd.read_csv(country_dir / "predictions.csv.gz") - --- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -623 passed, 5 skipped, 10 warnings in 50.54s -``` - -### App lint and tests - -Command, matching the app CI job: - -```text -bun run lint -bun test tests -``` - -Output: - -```text -$ eslint . --max-warnings=0 -bun test v1.3.12 (700fc117) - - 95 pass - 0 fail - 322 expect() calls -Ran 95 tests across 12 files. [1316.00ms] -``` - -### Production build - -`IS_WEBPACK_TEST=1 bun run build` completed successfully with Next.js 16.2.6 -using webpack. It compiled, type-checked, generated all 43 static pages, and -reported: - -```text -Route (app) -β”Œ β—‹ / -β”œ β—‹ /_not-found -β”œ β—‹ /expand -β”œ ● /model/[id] -β”‚ β”œ /model/gpt-5.6-sol -β”‚ β”œ /model/claude-fable-5.1 -β”‚ β”œ /model/kimi-k3 -β”‚ β”” [+30 more paths] -β”œ β—‹ /notes -β”œ ● /notes/[slug] -β”‚ β”œ /notes/2026-09-03-six-snap-households -β”‚ β”” /notes/2026-09-01-claude-fable-5-1-added -β”œ β—‹ /paper -β”œ β—‹ /robots.txt -β”” β—‹ /sitemap.xml - -β—‹ (Static) prerendered as static content -● (SSG) prerendered as static HTML (uses generateStaticParams) -``` - -## Delivery - -- Branch: `notes-route` -- Base: `add-fable-5-1` -- Local implementation commit before this report update: `725c1ac` -- Draft PR URL: unavailable because the remote branch could not be created. - -The remaining commands, once GitHub access is restored, are: - -```text -git push -u origin notes-route -gh pr create --draft --base add-fable-5-1 --head notes-route --title "Add /notes with two data-linked notes" --body "Adds dated, fact-tested notes for Claude Fable 5.1 and six shared SNAP denials, with committed pathway evidence." -``` - -## Could not do locally - -The default Turbopack path behind unprefixed `bun run build` fails in this -managed sandbox with `TurbopackInternalError: Failed to write app endpoint -/page`. Its cause chain ends in PostCSS loader evaluation trying to create a -process, bind a port, and receiving `Operation not permitted (os error 1)`. -`NEXT_TURBOPACK_USE_WORKER=0` reaches the same sandbox restriction. The webpack -build path completed successfully as recorded above. CI runs outside this -sandbox and can execute the default command. - -The requested push failed before authentication because the managed shell -cannot resolve GitHub: - -```text -fatal: unable to access 'https://github.com/PolicyEngine/policybench.git/': Could not resolve host: github.com -``` - -The installed GitHub connector was then tried as the network-independent -fallback. Both low-level tree creation and direct `notes-route` branch creation -returned `user cancelled MCP tool call`; a follow-up branch search confirmed -that `notes-route` does not exist remotely. Therefore no truthful PR URL can be -recorded and no draft PR can be opened from this environment. From 080ba3028251c388ed458ac6434c5f36dca755c7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 5 Sep 2026 12:38:51 -0400 Subject: [PATCH 09/10] Add the GPT-6 Astra note; pin each note's facts to its own data release - New note, 2026-09-05: GPT-6 Astra debuts second at 88.0% (#2 of 39). Row by row against GPT-5.6 Sol on the 1,973 scored outputs: both right on 1,816, both wrong on 95, 33 Astra-only misses against 29 Sol-only. Astra's solo misses cluster in two invented rules (an employer-sponsored insurance premium netted from wages, 13 rows; a disability flag read as Medicare eligibility, 10 rows). The row list with both predictions, the reference and the judge's note is committed (scripts/astra_vs_sol_rows.py regenerates it) and the test recomputes every fact from the frozen payload and annotations. - Notes keep the release their facts were checked against. The renderer shows "numbers from release X (board snapshot Y)" under the date; the test recomputes facts for notes on the frozen release and, for notes on a superseded release, checks the release is one the repository published. The frozen payload is read through snapshot_payload (gzip). Co-Authored-By: Claude Fable 5.1 --- app/src/components/NotesContent.tsx | 22 ++- .../2026-09-05-gpt-6-astra-debuts-second.json | 68 +++++++ app/src/notes/index.ts | 8 +- app/tests/notes.test.ts | 5 +- notes/data/astra_vs_sol_20260905.csv | 63 +++++++ .../data/astra_vs_sol_20260905.csv.meta.json | 8 + scripts/astra_vs_sol_rows.py | 175 ++++++++++++++++++ tests/test_notes.py | 141 +++++++++++++- 8 files changed, 476 insertions(+), 14 deletions(-) create mode 100644 app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json create mode 100644 notes/data/astra_vs_sol_20260905.csv create mode 100644 notes/data/astra_vs_sol_20260905.csv.meta.json create mode 100644 scripts/astra_vs_sol_rows.py diff --git a/app/src/components/NotesContent.tsx b/app/src/components/NotesContent.tsx index e690d2f..9329f0b 100644 --- a/app/src/components/NotesContent.tsx +++ b/app/src/components/NotesContent.tsx @@ -6,7 +6,7 @@ import { notes, type NoteFact, type PolicyBenchNote } from "../notes"; const PLACEHOLDER = /\{([A-Za-z][A-Za-z0-9]*)\}/g; export const NOTES_INTRO = - "Dated records of board changes and findings. Every number is checked against the frozen snapshot by a test in the repository."; + "Dated records of board changes and findings. Each note names the data release its numbers come from; a test in the repository checks every number against the frozen snapshot of that release."; export function formatNoteDate(date: string): string { return new Intl.DateTimeFormat("en-US", { @@ -134,12 +134,20 @@ export function NoteArticle({ return (
- + {title} diff --git a/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json new file mode 100644 index 0000000..af93d23 --- /dev/null +++ b/app/src/notes/2026-09-05-gpt-6-astra-debuts-second.json @@ -0,0 +1,68 @@ +{ + "slug": "2026-09-05-gpt-6-astra-debuts-second", + "date": "2026-09-05", + "title": "GPT-6 Astra debuts second: the two rules it invented", + "paragraphs": [ + "GPT-6 Astra reached the API on 2026-09-04 and joined the board on 2026-09-05 at {astraExact}% of answers within $1, #{astraRank} of {nModels}, behind GPT-5.6 Sol ({solExact}%) and ahead of Claude Fable 5.1 ({fableExact}%, #{fableRank}). Every row on this board is scored on {scoredOutputs} of its {totalOutputs} requested outputs: {excludedOutputs} outputs whose reference depends on an engine input the household facts never list are excluded for every model.", + "Row by row, the two OpenAI models agree far more than they differ. Both are right on {bothRight} scored outputs and both wrong on {bothWrong}. The gap is {astraOnlyMisses} outputs Astra misses and Sol gets right, against {solOnlyMisses} the other way.", + "Astra's solo misses cluster in two invented rules. In {esiRows} rows it treats a listed employer-sponsored insurance premium as a pre-tax salary reduction and subtracts it from wages before computing payroll tax, federal income tax or state income tax; the household facts state gross wages, and the reference keeps them. In {medicareRows} rows it marks a household member under 65 who is flagged disabled as Medicare-eligible; the reference grants Medicare before 65 only with 24 months of Social Security disability receipt recorded, and none of these households carry that input. The remaining {otherRows} solo misses are one-off rule errors with no shared mechanism. One further Medicare row of the same kind, on scenario_074, sits on an excluded output and is outside every count here.", + "The judge's row notes name an employer-sponsored insurance premium on {astraEsiMentions} of Astra's annotated rows and on {solEsiMentions} of Sol's. Sol netted the same premium once, on scenario_045, which cost it {solEsiRows} of its {solOnlyMisses} solo misses; the rest of Sol's solo misses share no mechanism.", + "The row list, with both predictions, the reference and the judge's note for every disagreement, is committed beside this note. Click any scenario in the explorer to see the household facts and each model's answer." + ], + "facts": { + "astraExact": 88.0, + "astraRank": 2, + "nModels": 39, + "solExact": 89.2, + "fableExact": 86.9, + "fableRank": 3, + "scoredOutputs": 1973, + "totalOutputs": 1984, + "excludedOutputs": 11, + "bothRight": 1816, + "bothWrong": 95, + "astraOnlyMisses": 33, + "solOnlyMisses": 29, + "esiRows": 13, + "medicareRows": 10, + "otherRows": 10, + "astraEsiMentions": 26, + "solEsiMentions": 4, + "solEsiRows": 2 + }, + "mentionRegexes": { + "esi": "employer[- ]sponsored insurance|\\bESI\\b" + }, + "data": [ + { + "label": "Dashboard data release", + "href": "https://github.com/PolicyEngine/policybench/releases/tag/dashboard-data-20260905c" + }, + { + "label": "Row list: every output one model gets right and the other misses", + "href": "https://github.com/PolicyEngine/policybench/blob/main/notes/data/astra_vs_sol_20260905.csv" + }, + { + "label": "Outputs excluded from scoring", + "href": "https://github.com/PolicyEngine/policybench/blob/main/paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/reference_exclusions.json" + }, + { + "label": "An ESI-premium row in the explorer (scenario_037 payroll tax)", + "href": "/?scenario=scenario_037&cell=payroll_tax~gpt-6-astra" + }, + { + "label": "A Medicare row in the explorer (scenario_008)", + "href": "/?scenario=scenario_008&cell=spouse_medicare_eligible~gpt-6-astra" + }, + { + "label": "GPT-6 Astra model page", + "href": "/model/gpt-6-astra" + }, + { + "label": "GPT-5.6 Sol model page", + "href": "/model/gpt-5.6-sol" + } + ], + "boardSnapshot": "2026-09-05", + "release": "dashboard-data-20260905c" +} diff --git a/app/src/notes/index.ts b/app/src/notes/index.ts index 60cccb2..851e342 100644 --- a/app/src/notes/index.ts +++ b/app/src/notes/index.ts @@ -1,5 +1,6 @@ import claudeFable51Added from "./2026-09-01-claude-fable-5-1-added.json"; import sixSnapHouseholds from "./2026-09-03-six-snap-households.json"; +import gpt6AstraDebutsSecond from "./2026-09-05-gpt-6-astra-debuts-second.json"; export type NoteFact = number | string | string[]; @@ -16,11 +17,14 @@ export type PolicyBenchNote = { facts: Record; mentionRegexes?: Record; data: NoteDataLink[]; - boardSnapshot: "2026-09-01"; - release: "dashboard-data-20260901c"; + // The board snapshot and data release the facts were checked against; a + // note keeps its own release when later releases move the board. + boardSnapshot: string; + release: string; }; export const notes = [ + gpt6AstraDebutsSecond, sixSnapHouseholds, claudeFable51Added, ] as PolicyBenchNote[]; diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index d01e031..c914cdc 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -35,8 +35,11 @@ describe("notes", () => { } }); - test("the notes page renders both titles without unresolved placeholders", () => { + test("the notes page renders every title without unresolved placeholders", () => { const markup = renderToStaticMarkup(createElement(NotesPageContent)); + expect(markup).toContain("GPT-6 Astra debuts second: the two rules it invented"); + expect(markup).toContain("release dashboard-data-20260905c"); + expect(markup).toContain("release dashboard-data-20260901c"); expect(markup).toContain("Six SNAP households the top three models deny"); expect(markup).toContain("Claude Fable 5.1 added"); expect(markup).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*\}/); diff --git a/notes/data/astra_vs_sol_20260905.csv b/notes/data/astra_vs_sol_20260905.csv new file mode 100644 index 0000000..cb1f3f8 --- /dev/null +++ b/notes/data/astra_vs_sol_20260905.csv @@ -0,0 +1,63 @@ +scenario_id,variable,direction,cluster,prediction_astra,prediction_sol,reference,judge_failure_source,judge_failure_subtype,judge_annotation +scenario_008,spouse_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model invoked a ""modeled disability-based Medicare eligibility rule"" that does not exist β€” the eligibility test is age 65 or older, and the Spouse is 41. It never checked the 24-month SSDI-receipt condition that governs real under-65 Medicare entitlement, which is unsatisfiable here since no Social Security disability income is listed, so its 1 should be 0." +scenario_015,head_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model invoked ""PolicyEngine's disability-based Medicare eligibility treatment"" solely because the head is listed as disabled; Medicare eligibility here is determined by age against the 65 threshold, and the disability route to Medicare requires 24 months of SSDI entitlement rather than a disability indicator. With the head aged 36, employed at $45,000 in wages, and receiving no disability insurance benefits, the correct evaluation returns False." +scenario_027,spouse_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model invoked a ""PolicyEngine disability-based Medicare eligibility rule"" that does not exist in the model β€” is_medicare_eligible tests the individual's own age β‰₯ 65 β€” and offered no check on whether the spouse held any disability-insurance entitlement. The spouse is 39 with $0 disability income, so the age test returns False." +scenario_039,head_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model routed eligibility through ""the stated disability"" alone, skipping the entitlement condition that actually opens the under-65 pathway: 24 months of Social Security disability insurance benefits, ESRD, or ALS. All are unlisted and therefore zero, so only the age-65 test applies and age 61 fails it." +scenario_075,head_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model claimed eligibility under ""the modeled Medicare disability pathway,"" but no disability pathway is modeled β€” PolicyEngine determines Medicare eligibility from the age-65 threshold alone, returning False for a 62-year-old. Even the statutory pathway would fail here, since it requires 24 months of SSDI entitlement and the household's Social Security disability income is $0." +scenario_079,spouse_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model asserted a bare ""disability-based Medicare eligibility rule"" with no derivation, mapping the disabled flag directly to eligibility. The correct derivation requires age 65+ (spouse is 55) or 24 months of SSDI entitlement (spouse receives no Social Security disability income); the model's answer is consistent with reading the disability and blindness booleans as a standalone eligibility trigger, which is the SSI/SSDI disability definition, not a Medicare entitlement condition." +scenario_088,head_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model asserted eligibility ""under PolicyEngine's disability-based Medicare eligibility rule because the head is disabled,"" treating the disability flag as sufficient rather than as one input to the SSDI pathway that additionally requires 24 months of disability-benefit entitlement. With the head aged 56, $0 SSDI income, and no ESRD/ALS fact, no Medicare pathway opens and the value is No." +scenario_093,dependent1_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model claimed eligibility ""under the modeled disability-based Medicare eligibility criterion because disability is reported,"" attributing a disability criterion to PolicyEngine that is_medicare_eligible does not implement. The only modeled criterion is age >= 65, which Dependent 1 at age 27 fails, and the reported disability flag carries no SSDI entitlement, ESRD, or ALS status on these facts." +scenario_111,spouse_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"Cited a ""PolicyEngine disability-based Medicare eligibility rule"" fired by the is_disabled flag alone. Medicare's under-65 route requires 24 months of SSDI entitlement or an ESRD/ALS diagnosis, none of which is present in this household, so the age-65 threshold governs and the 49-year-old spouse is not eligible." +scenario_116,spouse_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model asserted a 'modeled disability-based Medicare eligibility rule' that PolicyEngine does not have: is_medicare_eligible is keyed to the age-65 threshold, and the is_disabled flag drives SSI and the Medicaid aged/blind/disabled pathway, not Medicare. It mapped the disability boolean straight onto Medicare entitlement without requiring the SSDI receipt that the real 24-month pathway depends on, so it returned 1 for a 42-year-old with zero Social Security disability benefits." +scenario_037,federal_income_tax_before_refundable_credits,astra_only,esi_premium_netted_from_wages,952.1,1649.2,1649.2513427734375,llm_error,taxable_income_or_deductions,"It applied both the $16,100 standard deduction and the $9,629 overtime deduction correctly, but additionally subtracted the $5,789 employer-sponsored insurance premium from wages, which does not reduce the stated employment income, pushing taxable income down to $10,021 instead of $15,810.43. It also took a $50 saver's credit that is worth zero at AGI $41,539, above the 2026 single AGI ceiling for that credit." +scenario_037,payroll_tax,astra_only,esi_premium_netted_from_wages,2765.78,3208.64,3208.668701171875,llm_error,payroll_tax_base,"It correctly held that traditional 401(k) contributions and the FLSA overtime deduction leave payroll-tax wages untouched, then negated that by netting the $5,789 employer-sponsored insurance premium out of wages to reach a $36,154 base. The premium is not an employee section 125 salary reduction, so 7.65% applies to the full $41,943 for $3,208.67." +scenario_052,payroll_tax,astra_only,esi_premium_netted_from_wages,20558.11,20939.0,20939.0,llm_error,payroll_tax_base,"The model used the correct 2026 wage base ($184,500 Γ— 6.2% = $11,439) but treated the $16,208 employer-plan premiums as pretax and computed Medicare and the Additional Medicare Tax on $483,792, giving $7,014.98 and $2,104.13. PolicyEngine applies both to the full $500,000 of gross wages, producing $7,250 and $2,250; that single base reduction accounts for the entire $380.89 shortfall." +scenario_064,payroll_tax,astra_only,esi_premium_netted_from_wages,5940.0,7443.07,7443.0673828125,llm_error,payroll_tax_base,"The model applied 7.65% to $77,647 after removing the $19,648 employer-sponsored insurance premium as a pre-tax deduction. Employee Social Security and Medicare are owed on the head's full $97,295 of gross wages, making the answer short by 7.65% of the premium it excluded." +scenario_089,federal_income_tax_before_refundable_credits,astra_only,esi_premium_netted_from_wages,5885.06,6820.0,6819.7119140625,llm_error,taxable_income_or_deductions,"It had the full structure right β€” $32,200 standard deduction, $3,467.43 overtime deduction, Β§199A β€” but additionally excluded the $7,789 employee share of employer-sponsored insurance premiums as 'eligible pretax insurance,' which does not reduce the $58,934.98 of employment income already net of the 401(k) deferral. That single extra $7,789 deduction at 12% is $934.65, exactly its shortfall against $6,819.71." +scenario_089,payroll_tax,astra_only,esi_premium_netted_from_wages,4444.04,5040.0,5039.91015625,llm_error,payroll_tax_base,"It computed 7.65% of $58,092 after removing the $7,789 ESI premium as a pretax salary reduction. It was right that there is no Additional Medicare Tax and no mandatory NC employee payroll tax, but the base is the full $65,881 of gross wages, giving $5,039.91." +scenario_089,state_income_tax_before_refundable_credits,astra_only,esi_premium_netted_from_wages,2909.14,3220.0,3219.935302734375,llm_error,taxable_income_or_deductions,"It applied the correct $25,500 joint standard deduction and 3.99% rate to an AGI of $98,410.90, which is the correct $106,200.13 less the $7,789 employer-sponsored insurance premium. That premium is not netted out of the stated gross wages, so taxable income is $80,700.13." +scenario_091,federal_income_tax_before_refundable_credits,astra_only,esi_premium_netted_from_wages,652.9,1350.0,1349.998779296875,llm_error,taxable_income_or_deductions,"It cut AGI to $29,594 by deducting the $6,589 of employer-sponsored insurance premiums, which lie outside the $27,000 of employment income and are not an above-the-line deduction; correct AGI is $35,012.56. That single error shrinks its ordinary base to $7,728 against the true $14,316.66, even though its $16,100 standard deduction, $627.91 overtime deduction, and $119.90 saver's credit were all right." +scenario_091,payroll_tax,astra_only,esi_premium_netted_from_wages,1561.44,2065.5,2065.5,llm_error,payroll_tax_base,"The model explicitly recharacterized the $6,589 employer-sponsored insurance figure as pre-tax employee premiums and applied 7.65% to the resulting $20,411. That input does not reduce Social Security and Medicare wages, which remain the stated $27,000, so the tax is $1,674.00 + $391.50 = $2,065.50; the model's other two calls (401(k) contributions stay FICA-taxable, Wisconsin adds no employee payroll tax) were correct." +scenario_093,payroll_tax,astra_only,esi_premium_netted_from_wages,10544.22,12469.5,12469.5,llm_error,payroll_tax_base,"The model applied 7.65% to $137,833 β€” gross wages less the three $8,389 employer-sponsored insurance premiums treated as pre-tax β€” for $10,544.22. Its handling of the wage ceiling, the Additional Medicare Tax, and the absence of a Missouri employee payroll tax was right; the single error is removing ESI premiums from a payroll tax base that PolicyEngine keeps at the full $163,000." +scenario_110,payroll_tax,astra_only,esi_premium_netted_from_wages,7122.99,7650.0,7650.0,llm_error,payroll_tax_base,"The model excluded the $6,889 employer-sponsored insurance premium as pretax and applied 7.65% to $93,111, correctly ruling out Additional Medicare Tax and Ohio employee payroll taxes but on the wrong base. The stated $100,000 of gross wages is the FICA base, yielding $6,200 Social Security and $1,450 Medicare for $7,650." +scenario_119,payroll_tax,astra_only,esi_premium_netted_from_wages,3718.74,4207.5,4207.5,llm_error,payroll_tax_base,"The model explicitly chose to treat the $6,389 employer-sponsored insurance premium as pretax and applied the 7.65% combined rate to $48,611. It got the two adjacent rules right β€” traditional 401(k) contributions do not reduce FICA wages and Virginia imposes no mandatory employee payroll tax β€” but the employer-paid ESI premium likewise leaves the base at $55,000, giving 7.65% x $55,000 = $4,207.50." +scenario_120,payroll_tax,astra_only,esi_premium_netted_from_wages,12890.69,13496.16,13496.1552734375,llm_error,payroll_tax_base,"The model assembled the right three levies β€” 6.2% Social Security, 1.45% Medicare, 0.5% CT paid leave β€” but applied them to ""estimated payroll-taxable wages of $158,168,"" removing the $7,429 employer-sponsored insurance premium from the $165,597 of wages. The base for all three is the full wage figure, making its answer 4.5% low across the board." +scenario_031,snap,astra_only,other,288.0,0.0,0.0,llm_error,categorical_eligibility,"Identified the decisive fact β€” no deductible shelter costs, leaving only the ~$207 standard deduction against $1,987.75/month of countable Social Security, pension, and IRA income β€” and then overrode it by paying the one-person minimum allotment on the strength of the 200%-of-poverty categorical-eligibility screen. California's MCE waives the gross income and asset tests but not the net income test, and net income of ~$1,780/month exceeds the ~$1,305 one-person limit, so the household is ineligible and the minimum benefit has nothing to floor." +scenario_057,state_refundable_credits,astra_only,other,23.24,33.2,33.20000076293945,llm_error,state_local_rule,"The model derived the federal EITC of $664 correctly but applied a 3.5% Louisiana match, the pre-2018 rate. Act 2 of the 2018 Second Extraordinary Session raised the credit to 5% of the federal EITC for tax years beginning on or after January 1, 2018, giving $33.20 instead of $23.24." +scenario_081,state_income_tax_before_refundable_credits,astra_only,other,8230.1,8238.7,8238.40625,llm_error,taxable_income_or_deductions,"It removed the $56 of taxable interest from the base by invoking the Massachusetts bank-interest exclusion, which does not reach this household's generic taxable interest, and offset the $110.12 dividend with the short-term capital loss. Both amounts are taxed at 5% β€” the interest in Part B and the dividend in Part A β€” so its otherwise correct 164,602 computation lands $8.31 short." +scenario_085,state_income_tax_before_refundable_credits,astra_only,other,51.42,0.0,0.0,llm_error,credit_phaseout,"The model reached the correct $51.42 gross tax but ruled out forgiveness because ""total household income precludes tax forgiveness,"" applying gross household income instead of Schedule SP eligibility income. Schedule SP excludes Social Security and Railroad Retirement benefits and old-age distributions from eligible retirement plans, so eligibility income is $1,675, forgiveness is 100%, and the liability is $0." +scenario_095,state_refundable_credits,astra_only,other,119.38,0.0,0.0,llm_error,state_local_rule,"Built the NJ EITC off a self-constructed ""age-adjusted federal credit"" instead of the federal EITC as computed; the state credit is 40% of the federal EITC amount, which is $0 here because the no-qualifying-child EITC is unavailable above age 64 and this filer is 73. It compounded the error by using self-employment income less half the self-employment tax as the earned income base β€” that subtraction belongs to AGI, not to EITC earned income β€” and its $119.38 is exactly 40% of a roughly $298 childless credit phased down against AGI of about $15,300, a credit the age ceiling eliminates entirely." +scenario_100,federal_refundable_credits,astra_only,other,822.4,2878.25,2878.0966796875,llm_error,taxable_income_or_deductions,"It based the EITC on $2,056 of wages after subtracting the $3,859 traditional 401(k) contribution, producing $822.40. The elective deferral lowers AGI but leaves the earned-income base at $5,915, so the EITC is $2,365.89 and earnings clear the $2,500 refundable-CTC threshold for a further $512.21." +scenario_101,federal_income_tax_before_refundable_credits,astra_only,other,13198.6,13418.6,13418.599609375,llm_error,taxable_income_or_deductions,"Its AGI, standard deduction, and 2026 brackets are all correct, but it subtracted an additional $1,000 auto-loan-interest deduction ($1,400 less the $400 phaseout for MAGI above $100,000) to reach $84,030 of taxable income instead of $85,030. The deduction is unavailable because the qualified-vehicle status flags are unlisted and therefore false, and 22% of the $1,000 is the entire $220 error." +scenario_118,state_refundable_credits,astra_only,other,75.0,375.0,375.0,llm_error,thresholds_rates,"The model found the right credit and confirmed property taxes were sufficient to reach the cap, then used the $75 ceiling that applies only when no household member is 65 or older. The head is 74, so Β§606(e) sets the maximum at $375, and that ceiling binds whether household gross income is the $2,800 of Social Security alone or the grossed-up figure the model assumed." +scenario_119,state_refundable_credits,astra_only,other,524.35,0.0,0.0,llm_error,state_local_rule,"Booked 20% of the federal EITC as a refundable Virginia credit, conflating the nonrefundable 20% credit with the separate 15% refundable election that expired for taxable years beginning on or after January 1, 2026. Its federal EITC input of $2,621.73 is also far above the phased-down two-child credit at ~$55,800 of AGI, compounding a rule error with a phaseout error." +scenario_123,payroll_tax,astra_only,other,11179.5,11194.0,11194.0,llm_error,state_local_rule,"It structured the answer correctly β€” $8,990 Social Security, $2,102.50 Medicare, plus a Pennsylvania employee unemployment contribution β€” but priced the PA piece at $87, which is 0.06% of $145,000, a superseded rate. The current Pennsylvania employee UC withholding rate is 0.07%, giving $101.50 and a total of $11,194." +scenario_045,federal_refundable_credits,sol_only,esi_premium_netted_from_wages,0.0,345.0,0.0,llm_error,taxable_income_or_deductions,"The model explicitly subtracted $21,208 of insurance premiums from gross wages as though they were pre-tax, yielding $15,068 of EITC income. Because that pre-tax treatment was not supplied, income remains $36,276 and the childless EITC is fully phased out." +scenario_045,payroll_tax,sol_only,esi_premium_netted_from_wages,2775.11,1153.0,2775.08251953125,llm_error,payroll_tax_base,"The model applied 7.65% to ""15068 in wages after pre-tax employer insurance premiums,"" deducting the whole $21,208 ESI premium from the FICA base. PolicyEngine taxes the full $36,276 of employment income, giving $2,249.09 of Social Security tax and $526.00 of Medicare tax for $2,775.08." +scenario_007,federal_income_tax_before_refundable_credits,sol_only,other,3707.7,3737.0,3707.695068359375,llm_error,taxable_income_or_deductions,"It added the $284 of tax-exempt private pension into provisional income; Β§86 adds back tax-exempt interest only, and the engine's provisional income excludes tax-exempt pension. That raised taxable benefits by $241.40 to $14,025.52 and taxable income to $33,205.52, producing $3,736.66, which it rounded to $3,737." +scenario_008,federal_refundable_credits,sol_only,other,12433.61,12476.0,12433.611328125,llm_error,taxable_income_or_deductions,"It named the $8,231 three-or-more-child EITC correctly but computed the earned-income-limited refundable CTC as $4,245, which is 15% x ($30,800 - $2,500) using gross self-employment income. Subtracting the deductible half of SE tax ($282.59) sets earned income at $30,517.41 and the refundable CTC at $4,202.61." +scenario_012,federal_refundable_credits,sol_only,other,6127.0,6227.0,6127.0,llm_error,thresholds_rates,"The model nailed the EITC leg at the correct $4,427 one-child maximum but applied an $1,800 per-child refundable CTC cap; the 2026 refundable ceiling is $1,700. The full $100 error is that single misremembered cap." +scenario_013,state_refundable_credits,sol_only,other,25.0,0.0,25.0,llm_error,state_local_rule,"The model required ""qualifying property tax, rent, dependents, or other credit-generating facts,"" importing the Form 140PTC and family income tax credit conditions. The increased excise tax credit imposes none of those and grants $25 to this resident single filer with $6,736 of federal AGI." +scenario_023,head_medicaid_eligible,sol_only,other,1.0,0.0,1.0,llm_error,categorical_eligibility,"It correctly computed MAGI of $22,535 above the expansion limit and correctly noted that disability alone does not create SSI-linked eligibility, but it never tested the pathway that disability plus earnings does create. The working-disabled buy-in category requires employment rather than SSI receipt and carries a 250% FPL income limit, which the head clears at 141% FPL." +scenario_026,child1_medicaid_eligible,sol_only,other,1.0,0.0,1.0,llm_error,state_local_rule,"It claimed MAGI exceeds North Carolina's limit for an 11-year-old without stating either figure. MAGI is $85,209 of wages less $2,908 of traditional 401(k)/IRA contributions = $82,301, which is 213% FPL for five people and under NC's ~216% FPL older-child limit." +scenario_026,child2_medicaid_eligible,sol_only,other,1.0,0.0,1.0,llm_error,thresholds_rates,"It asserted MAGI exceeds North Carolina's limit for an 11-year-old without computing either quantity. MAGI is $82,301 β€” wages less the traditional 401(k) and IRA contributions β€” which is 2.13x the 2026 five-person guideline and inside NC's 211%-plus older-child limit; the ineligible answer is consistent with applying the federal 133% floor." +scenario_026,child3_medicaid_eligible,sol_only,other,1.0,0.0,1.0,llm_error,thresholds_rates,"Asserted MAGI exceeds North Carolina's limit for a 9-year-old without stating either number. MAGI is $82,301 β€” 213% of the ~$38,600 five-person 2026 guideline β€” against the OLDER_CHILD limit of 211% FPL plus the 5-percentage-point disregard, so the answer tracks the 133% ACA minimum instead of NC's level." +scenario_026,federal_refundable_credits,sol_only,other,2013.4,2058.76,2013.4892578125,llm_error,taxable_income_or_deductions,"It reached $2,058.76 by deducting the $378 of auto loan interest in addition to the $32,200 standard deduction and the $7,746 overtime deduction, giving taxable income of $41,977 and tax of $4,541.24. The OBBBA vehicle-loan interest deduction requires a qualifying new, US-final-assembly vehicle financed after 2024 β€” conditions unlisted here and therefore false β€” so taxable income is $42,355, tax is $4,586.51, and the refundable remainder is $2,013.49, exactly $45.27 below its answer." +scenario_027,snap,sol_only,other,288.0,0.0,287.68316650390625,llm_error,categorical_eligibility,"It denied the household because SNAP net income sits above ""the applicable two-person limit,"" applying a net income test that Connecticut's broad-based categorical eligibility waives for this unit. Past that test, the $546 maximum allotment minus the $635.10 expected contribution is negative, so the minimum allotment of $23.84/$24.37 per month is paid." +scenario_028,state_refundable_credits,sol_only,other,62.41,0.0,62.41279220581055,llm_error,state_local_rule,"The model returned a one-line denial that any PA refundable individual income tax credit applies, performing no federal-EITC computation to feed the state match. Completing that step gives 10% of $624.13, or $62.41." +scenario_038,state_income_tax_before_refundable_credits,sol_only,other,0.0,282.0,0.0,llm_error,taxable_income_or_deductions,"The model applied Louisiana’s 3% rate to an implied positive base of about $9,400 after an unspecified married deduction. The applicable joint-filer and two-dependent calculation leaves zero taxable income, so the 3% rate has no positive base to tax." +scenario_044,state_refundable_credits,sol_only,other,0.0,250.0,0.0,llm_error,categorical_eligibility,"The model incorrectly placed the $250 Kansas food sales tax credit in the refundable-credit total. Kansas treats this credit as nonrefundable, leaving state_refundable_credits at $0." +scenario_045,head_medicaid_eligible,sol_only,other,0.0,1.0,0.0,llm_error,taxable_income_or_deductions,"The model's 'estimated MAGI of 15068' is the $36,276 wage net of the $21,208 employer-sponsored insurance premium, treating a health-coverage expense input as an income exclusion; Medicaid MAGI here is $36,276, or 2.27 x FPL, well above Michigan's 138% FPL expansion-adult limit of about $22,055. The expansion pathway it cited is therefore unavailable, and no non-MAGI pathway applies to a 44-year-old with no SSI and no disability." +scenario_062,federal_income_tax_before_refundable_credits,sol_only,other,0.0,4328.0,0.0,llm_error,taxable_income_or_deductions,"It listed survivor benefits as taxable income alongside 85%-included Social Security, producing AGI of about $63,245 and roughly $38,100 of taxable income after the standard, age, senior, and QBI deductions. The $28,800 survivor-benefits input does not enter countable income, which is $37,642; at a $26,986 provisional income only $993 of Social Security is taxable, giving AGI of $17,323 against ~$24,150 of deductions and $0 of tax." +scenario_064,child1_chip_eligible,sol_only,other,0.0,1.0,0.0,llm_error,health_coverage,"It correctly placed the child above Wisconsin's Medicaid ceiling for ages 6–18 but then asserted membership in the ""upper CHIP income band"" without computing MAGI against the poverty guideline. That band tops out at 306% FPL, and the tax unit's MAGI of about $130,700 β€” wages plus the $20,000 retirement distribution plus $33,350 of pass-through income, less the farm loss and the $3,000 capital-loss cap β€” is roughly 340% of the ~$38,400 five-person guideline, so the child overshoots the band rather than landing in it." +scenario_064,dependent2_chip_eligible,sol_only,other,0.0,1.0,0.0,llm_error,age_disability,"The model built its answer on a two-tier income band β€” 'above the Medicaid ceiling but within Wisconsin's upper CHIP income band' β€” after granting that dependent2 is 'under 19' and therefore inside the child pathway. PolicyEngine places age 18 above the CHIP child category's age ceiling, so dependent2 never enters the Medicaid-to-CHIP income band the model reasoned about; the trace shows Medicaid category NONE and is_chip_eligible False." +scenario_067,state_income_tax_before_refundable_credits,sol_only,other,2486.91,2442.66,2486.908935546875,llm_error,taxable_income_or_deductions,"Used the correct 2.95% 2026 rate but subtracted $4,500 of exemptions by adding a 'qualifying-child' amount for a 23-year-old. Indiana's additional $1,500 exemption is limited to a dependent child under 19, or under 24 and a full-time student, so total exemptions are $3,000 and the base is $84,302." +scenario_072,state_income_tax_before_refundable_credits,sol_only,other,0.0,356.0,0.0,llm_error,state_local_rule,"It applied only two estimated personal exemptions to the $20,166 remaining after excluding Social Security and veterans benefits, then taxed the balance at 4.25%. Michigan's senior standard deduction of $40,000 on a joint return where both filers have reached 67 applies to that same $20,166 of wage, dividend, and interest income and reduces Michigan taxable income to $0." +scenario_073,snap,sol_only,other,288.0,0.0,287.68316650390625,llm_error,thresholds_rates,"It concluded the income is too high to produce a positive allotment after deductions and the 30% benefit reduction. That computation matches the engine β€” $298 maximum minus a $526.50 expected contribution β€” but the result is floored at the one/two-person minimum allotment of 8% of the maximum, $23.84/mo, not zero." +scenario_079,state_refundable_credits,sol_only,other,50.0,0.0,50.0,llm_error,state_local_rule,"The model found no fact generating an Arizona refundable credit, overlooking that residency plus low AGI is the entire eligibility test for the increased excise tax credit. It pays $25 for each member of the tax unit up to $100, so this married couple receives $50." +scenario_095,state_income_tax_before_refundable_credits,sol_only,other,0.0,28.14,0.0,llm_error,state_local_rule,"The model applied the Social Security exclusion, the age-qualified retirement income exclusion for the $11,385 distribution, and the personal, age, and medical deductions, then ran the leftover self-employment income through the 1.4% bracket for $28.14. It omitted New Jersey's statutory zero-tax provision (N.J.S.A. 54A:2-1.1), under which NJ gross income of $4,198 β€” below the $20,000 surviving-spouse threshold and the $10,000 single threshold β€” produces no tax liability at all, so the value is $0.00." +scenario_098,snap,sol_only,other,0.0,3576.0,0.0,llm_error,taxable_income_or_deductions,"The model incorrectly treated the large rent and shelter deductions as reducing SNAP net income to zero. The household fails the income eligibility requirement before those deductions can support a benefit, so awarding the full maximum allotment was erroneous." +scenario_099,payroll_tax,sol_only,other,14767.5,14602.5,14767.5,llm_error,thresholds_rates,"Reproduced Social Security ($10,230) and Medicare ($2,392.50) exactly but estimated CA SDI at $1,980, i.e. the 2025 rate of 1.2%, rather than $2,145 at the 2026 rate of 1.3%. The entire $165 miss is the SDI rate parameter." +scenario_108,snap,sol_only,other,288.0,0.0,287.68316650390625,llm_error,categorical_eligibility,"It denied the benefit because countable net income stays above the one-person elderly-or-disabled net income limit, missing Wisconsin's broad-based categorical eligibility via a TANF-funded non-cash benefit, which screens gross income at 200% FPG β€” passed here at 1.98 times the $1,304.17 guideline β€” and waives that net income test and the asset test. The eligible household then receives the minimum allotment of $23.84/month, $24.37 from October, totaling $287.68." +scenario_109,federal_refundable_credits,sol_only,other,12038.09,12012.0,12038.0859375,llm_error,taxable_income_or_deductions,"The model paired the correct $8,231 EITC with an ACTC of $3,781, computed as 15% of net SE earnings ($30,000 Γ— 0.9235 = $27,705) above $2,500. The correct earned-income base is gross SE income less half the SE tax, $27,880.57, giving $3,807.09 β€” the 92.35% haircut belongs to the SE-tax computation, not to the ACTC earned-income base." +scenario_121,head_medicare_eligible,sol_only,other,0.0,1.0,0.0,llm_error,age_disability,"It explicitly overrode the age test β€” ""eligible ... under the disability pathway even though the head is under age 65"" β€” asserting a pre-65 route that PolicyEngine does not implement for is_medicare_eligible. The disability_benefits input is not SSDI and carries no entitlement-duration information, leaving the age-65 comparison as the only operative condition, which the head fails at 53." diff --git a/notes/data/astra_vs_sol_20260905.csv.meta.json b/notes/data/astra_vs_sol_20260905.csv.meta.json new file mode 100644 index 0000000..9382695 --- /dev/null +++ b/notes/data/astra_vs_sol_20260905.csv.meta.json @@ -0,0 +1,8 @@ +{ + "release": "dashboard-data-20260905c", + "source_run": "us_full_run_20260612_policyengine_4_16_1_populace", + "annotations_sha256": "735b75d5a6599c8375a97f000b93c1904d6f27588480300bee0ab22430b893ef", + "rows": 62, + "generated_at_utc": "2026-09-05T16:36:43.512612+00:00", + "script": "scripts/astra_vs_sol_rows.py" +} diff --git a/scripts/astra_vs_sol_rows.py b/scripts/astra_vs_sol_rows.py new file mode 100644 index 0000000..fba95ba --- /dev/null +++ b/scripts/astra_vs_sol_rows.py @@ -0,0 +1,175 @@ +"""Row-level comparison of GPT-6 Astra and GPT-5.6 Sol on the frozen board. + +Writes ``notes/data/astra_vs_sol_20260905.csv``: every scored output one of +the two models gets right and the other misses, with both predictions, the +reference, the judge's row annotation, and a cluster label for Astra's two +repeated mechanisms. ``tests/test_notes.py`` regenerates the rows and +compares, so the note's counts stay tied to the frozen payload. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + +from policybench.snapshot_payload import read_run_payload + +ROOT = Path(__file__).resolve().parents[1] +RUN_DIR = ( + ROOT + / "paper/snapshot/20260501/runs" + / "us_full_run_20260612_policyengine_4_16_1_populace" +) +ANNOTATIONS = ( + ROOT + / "annotations/us_full_run_20260612_policyengine_4_16_1_populace" + / "us_audit_row_annotations.csv" +) +OUTPUT = ROOT / "notes/data/astra_vs_sol_20260905.csv" +SOL, ASTRA = "gpt-5.6-sol", "gpt-6-astra" + +# Astra's two repeated mechanisms, identified from the judge annotations: +# an employer-sponsored insurance premium netted out of wages before a tax +# base, and a disability flag read as Medicare eligibility before 65. +ESI_ROWS = { + ("scenario_037", "payroll_tax"), + ("scenario_052", "payroll_tax"), + ("scenario_064", "payroll_tax"), + ("scenario_089", "payroll_tax"), + ("scenario_091", "payroll_tax"), + ("scenario_093", "payroll_tax"), + ("scenario_110", "payroll_tax"), + ("scenario_119", "payroll_tax"), + ("scenario_120", "payroll_tax"), + ("scenario_037", "federal_income_tax_before_refundable_credits"), + ("scenario_089", "federal_income_tax_before_refundable_credits"), + ("scenario_091", "federal_income_tax_before_refundable_credits"), + ("scenario_089", "state_income_tax_before_refundable_credits"), +} +# Sol netted the same premium once, on scenario_045 (payroll tax and the +# refundable credits that key off earned income). +SOL_ESI_ROWS = { + ("scenario_045", "payroll_tax"), + ("scenario_045", "federal_refundable_credits"), +} +MEDICARE_SCENARIOS = { + "scenario_008", + "scenario_015", + "scenario_027", + "scenario_039", + "scenario_074", + "scenario_075", + "scenario_079", + "scenario_088", + "scenario_093", + "scenario_111", + "scenario_116", +} +CLUSTER_ESI = "esi_premium_netted_from_wages" +CLUSTER_MEDICARE = "disability_read_as_medicare" +CLUSTER_OTHER = "other" + +FIELDS = [ + "scenario_id", + "variable", + "direction", + "cluster", + "prediction_astra", + "prediction_sol", + "reference", + "judge_failure_source", + "judge_failure_subtype", + "judge_annotation", +] + + +def _hit(record: dict) -> bool: + return record.get("exact") == 100 + + +def cluster_for(scenario_id: str, variable: str, direction: str) -> str: + key = (scenario_id, variable) + if direction == "astra_only": + if key in ESI_ROWS: + return CLUSTER_ESI + if scenario_id in MEDICARE_SCENARIOS and "medicare" in variable: + return CLUSTER_MEDICARE + return CLUSTER_OTHER + return CLUSTER_ESI if key in SOL_ESI_ROWS else CLUSTER_OTHER + + +def comparison_rows(payload: dict, annotations: dict) -> list[dict]: + rows = [] + for scenario_id, variables in payload["scenarioPredictions"].items(): + for variable, models in variables.items(): + astra, sol = models.get(ASTRA), models.get(SOL) + if not astra or not sol or astra.get("scored") is False: + continue + astra_hit, sol_hit = _hit(astra), _hit(sol) + if astra_hit == sol_hit: + continue + direction = "astra_only" if sol_hit else "sol_only" + judged = annotations.get( + (ASTRA if direction == "astra_only" else SOL, scenario_id, variable), + {}, + ) + rows.append( + { + "scenario_id": scenario_id, + "variable": variable, + "direction": direction, + "cluster": cluster_for(scenario_id, variable, direction), + "prediction_astra": astra["prediction"], + "prediction_sol": sol["prediction"], + "reference": astra["groundTruth"], + "judge_failure_source": judged.get("failure_source", ""), + "judge_failure_subtype": judged.get("failure_subtype", ""), + "judge_annotation": judged.get("annotation", ""), + } + ) + rows.sort( + key=lambda row: ( + row["direction"], + row["cluster"], + row["scenario_id"], + row["variable"], + ) + ) + return rows + + +def load_annotations(path: Path = ANNOTATIONS) -> dict: + with path.open(encoding="utf-8", newline="") as source: + return { + (row["model"], row["scenario_id"], row["variable"]): row + for row in csv.DictReader(source) + } + + +def main() -> None: + payload = read_run_payload(RUN_DIR) + rows = comparison_rows(payload, load_annotations()) + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + with OUTPUT.open("w", encoding="utf-8", newline="") as sink: + writer = csv.DictWriter(sink, fieldnames=FIELDS) + writer.writeheader() + writer.writerows(rows) + meta = { + "release": "dashboard-data-20260905c", + "source_run": RUN_DIR.name, + "annotations_sha256": hashlib.sha256(ANNOTATIONS.read_bytes()).hexdigest(), + "rows": len(rows), + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "script": "scripts/astra_vs_sol_rows.py", + } + OUTPUT.with_suffix(OUTPUT.suffix + ".meta.json").write_text( + json.dumps(meta, indent=2) + "\n" + ) + print(f"wrote {OUTPUT.relative_to(ROOT)} ({len(rows)} rows)") + + +if __name__ == "__main__": + main() diff --git a/tests/test_notes.py b/tests/test_notes.py index 89eead6..e1c6b9a 100644 --- a/tests/test_notes.py +++ b/tests/test_notes.py @@ -12,13 +12,14 @@ import pytest +from policybench.snapshot_payload import read_run_payload + ROOT = Path(__file__).resolve().parents[1] NOTES_DIR = ROOT / "app/src/notes" RUN_DIR = ( ROOT / "paper/snapshot/20260501/runs/" "us_full_run_20260612_policyengine_4_16_1_populace" ) -DATA_PATH = RUN_DIR / "data.json" PREDICTIONS_PATH = RUN_DIR / "predictions.csv.gz" REFERENCES_PATH = RUN_DIR / "reference_outputs.csv" REFERENCE_META_PATH = RUN_DIR / "reference_outputs.csv.meta.json" @@ -29,6 +30,8 @@ CLAUDE_NOTE = "2026-09-01-claude-fable-5-1-added" SNAP_NOTE = "2026-09-03-six-snap-households" +ASTRA_NOTE = "2026-09-05-gpt-6-astra-debuts-second" +ASTRA_ROWS_PATH = ROOT / "notes/data/astra_vs_sol_20260905.csv" TOP_MODELS = ("gpt-5.6-sol", "claude-fable-5.1", "kimi-k3") PLACEHOLDER = re.compile(r"\{([A-Za-z][A-Za-z0-9]*)\}") @@ -46,7 +49,20 @@ def _note(slug: str) -> dict: @cache def _dashboard() -> dict: - return _load_json(DATA_PATH) + return read_run_payload(RUN_DIR) + + +@cache +def _frozen_release() -> str: + manifest = _load_json(ROOT / "paper/snapshot/20260501/manifest.json") + return manifest["published_dashboard_artifact"]["tag"] + + +# A note keeps the release its facts were checked against. Facts of a note on +# the frozen release are recomputed here; a note on a superseded release keeps +# the facts verified when that release was frozen (git history holds the run). +SUPERSEDED_RELEASES = {"dashboard-data-20260901c": "2026-09-01"} +CURRENT_RELEASE_SNAPSHOT = "2026-09-05" @cache @@ -90,8 +106,11 @@ def test_note_schema_and_placeholders(path: Path) -> None: note = _load_json(path) assert note["slug"] == path.stem assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", note["date"]) - assert note["boardSnapshot"] == "2026-09-01" - assert note["release"] == "dashboard-data-20260901c" + if note["release"] == _frozen_release(): + assert note["boardSnapshot"] == CURRENT_RELEASE_SNAPSHOT + else: + assert note["boardSnapshot"] == SUPERSEDED_RELEASES[note["release"]] + assert note["boardSnapshot"] <= note["date"] or note["release"] != _frozen_release() assert note["paragraphs"] assert note["data"] @@ -103,8 +122,21 @@ def test_note_schema_and_placeholders(path: Path) -> None: assert placeholders == set(note["facts"]) +def _recompute_against_frozen_snapshot(note: dict) -> bool: + """Whether the note's facts are recomputed here: only when its release is + the frozen snapshot's. A note on a superseded release keeps the facts that + were verified when that release was frozen; the test then checks the + release is one the repository has published.""" + if note["release"] == _frozen_release(): + return True + assert note["release"] in SUPERSEDED_RELEASES, note["release"] + return False + + def test_claude_fable_note_facts() -> None: note = _note(CLAUDE_NOTE) + if not _recompute_against_frozen_snapshot(note): + return board_rows = [ row for row in _dashboard()["modelStats"] if row["condition"] == "no_tools" ] @@ -145,6 +177,8 @@ def _mention_count(rows: list[dict[str, str]], pattern: str) -> int: def test_six_snap_households_note_facts() -> None: note = _note(SNAP_NOTE) + if not _recompute_against_frozen_snapshot(note): + return references = _snap_references() eligible_ids = { scenario_id for scenario_id, value in references.items() if value > 0 @@ -243,3 +277,102 @@ def test_categorical_asset_error_statements() -> None: assert sum(error <= 0.01 for error in sol_errors) == 3 assert sum(0.01 < error <= 0.10 for error in sol_errors) == 1 assert sum(error <= 0.01 for error in fable_errors) == 4 + + +def _judge_annotations() -> dict[tuple[str, str, str], dict[str, str]]: + path = ( + ROOT + / "annotations/us_full_run_20260612_policyengine_4_16_1_populace" + / "us_audit_row_annotations.csv" + ) + with path.open(encoding="utf-8", newline="") as source: + return { + (row["model"], row["scenario_id"], row["variable"]): row + for row in csv.DictReader(source) + } + + +def test_astra_note_facts() -> None: + sys.path.insert(0, str(ROOT / "scripts")) + from astra_vs_sol_rows import ( + ASTRA, + CLUSTER_ESI, + CLUSTER_MEDICARE, + CLUSTER_OTHER, + SOL, + comparison_rows, + ) + + note = _note(ASTRA_NOTE) + assert note["release"] == _frozen_release() + payload = _dashboard() + annotations = _judge_annotations() + + expected_rows = comparison_rows(payload, annotations) + with ASTRA_ROWS_PATH.open(encoding="utf-8", newline="") as source: + committed = list(csv.DictReader(source)) + assert [ + (r["scenario_id"], r["variable"], r["direction"], r["cluster"]) + for r in committed + ] == [ + (r["scenario_id"], r["variable"], r["direction"], r["cluster"]) + for r in expected_rows + ] + + board_rows = [r for r in payload["modelStats"] if r["condition"] == "no_tools"] + by_model = {r["model"]: r for r in board_rows} + + def rank(model: str) -> int: + return 1 + sum(r["exact"] > by_model[model]["exact"] for r in board_rows) + + both_right = both_wrong = 0 + for variables in payload["scenarioPredictions"].values(): + for models in variables.values(): + astra, sol = models.get(ASTRA), models.get(SOL) + if not astra or not sol or astra.get("scored") is False: + continue + hits = (astra.get("exact") == 100, sol.get("exact") == 100) + both_right += hits == (True, True) + both_wrong += hits == (False, False) + astra_only = [r for r in expected_rows if r["direction"] == "astra_only"] + sol_only = [r for r in expected_rows if r["direction"] == "sol_only"] + excluded = { + (e["scenarioId"], e["variable"]) for e in payload["referenceExclusions"] + } + regex = re.compile(note["mentionRegexes"]["esi"], re.IGNORECASE) + + def mentions(model: str) -> int: + return sum( + bool(regex.search(row["annotation"] or "")) + for key, row in annotations.items() + if key[0] == model + ) + + derived = { + "astraExact": _display_one_decimal(by_model[ASTRA]["exact"]), + "astraRank": rank(ASTRA), + "nModels": len(board_rows), + "solExact": _display_one_decimal(by_model[SOL]["exact"]), + "fableExact": _display_one_decimal(by_model["claude-fable-5.1"]["exact"]), + "fableRank": rank("claude-fable-5.1"), + "scoredOutputs": by_model[ASTRA]["n"], + "totalOutputs": by_model[ASTRA]["n"] + len(excluded), + "excludedOutputs": len(excluded), + "bothRight": both_right, + "bothWrong": both_wrong, + "astraOnlyMisses": len(astra_only), + "solOnlyMisses": len(sol_only), + "esiRows": sum(r["cluster"] == CLUSTER_ESI for r in astra_only), + "medicareRows": sum(r["cluster"] == CLUSTER_MEDICARE for r in astra_only), + "otherRows": sum(r["cluster"] == CLUSTER_OTHER for r in astra_only), + "astraEsiMentions": mentions(ASTRA), + "solEsiMentions": mentions(SOL), + "solEsiRows": sum(r["cluster"] == CLUSTER_ESI for r in sol_only), + } + assert note["facts"] == derived + # The excluded Medicare row the note mentions is real and outside the rows. + assert ("scenario_074", "head_medicare_eligible") in excluded + assert not any( + r["scenario_id"] == "scenario_074" and "medicare" in r["variable"] + for r in expected_rows + ) From 1a19eafe11a931ab8723dc7f4064286c8870b7cd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 5 Sep 2026 12:54:40 -0400 Subject: [PATCH 10/10] Show whole-number board rates with one decimal on notes; plain-case release line Co-Authored-By: Claude Fable 5.1 --- app/src/components/NotesContent.tsx | 17 +++++++++++++---- app/tests/notes.test.ts | 3 +++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/app/src/components/NotesContent.tsx b/app/src/components/NotesContent.tsx index 9329f0b..fe48f20 100644 --- a/app/src/components/NotesContent.tsx +++ b/app/src/components/NotesContent.tsx @@ -17,11 +17,18 @@ export function formatNoteDate(date: string): string { }).format(new Date(`${date}T00:00:00Z`)); } +// Facts named *Exact or *Rate are board percentages shown to one decimal, so +// a whole-number rate (88.0) keeps its decimal like every other row. +const ONE_DECIMAL_FACT = /(Exact|Rate)$/; + function factText(key: string, fact: NoteFact): string { if (Array.isArray(fact)) return fact.join(", "); if (key === "referenceAnnual" && typeof fact === "number") { return String(Math.round(fact)); } + if (ONE_DECIMAL_FACT.test(key) && typeof fact === "number") { + return fact.toFixed(1); + } if (typeof fact === "number" && Number.isInteger(fact)) { return fact.toLocaleString("en-US"); } @@ -134,14 +141,16 @@ export function NoteArticle({ return (
-
- +
+ - numbers from{" "} + Numbers from{" "} release {note.release} {" "} diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts index c914cdc..6f042fa 100644 --- a/app/tests/notes.test.ts +++ b/app/tests/notes.test.ts @@ -39,6 +39,9 @@ describe("notes", () => { const markup = renderToStaticMarkup(createElement(NotesPageContent)); expect(markup).toContain("GPT-6 Astra debuts second: the two rules it invented"); expect(markup).toContain("release dashboard-data-20260905c"); + // Whole-number board rates keep one decimal. + expect(markup).toContain("at 88.0% of answers within $1"); + expect(markup).not.toContain("at 88% of answers"); expect(markup).toContain("release dashboard-data-20260901c"); expect(markup).toContain("Six SNAP households the top three models deny"); expect(markup).toContain("Claude Fable 5.1 added");