diff --git a/.gitignore b/.gitignore index e1b5f8b..a4b63cf 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ results/ground_truth.csv results/scenarios.csv output/ data/ +!notes/data/ +!notes/data/** # Committed sensitivity evidence (pinned by tests/test_sensitivity_evidence.py) !sensitivity/data/ diff --git a/app/src/App.tsx b/app/src/App.tsx index c118533..a727362 100644 --- a/app/src/App.tsx +++ b/app/src/App.tsx @@ -1,5 +1,6 @@ "use client"; +import Link from "next/link"; import { useCallback, useEffect, @@ -353,6 +354,10 @@ export default function App() { availableViews={availableViews} navItems={navItems} activeNav={activeNav} + actionLinks={[ + { label: "Paper", href: "/paper", type: "internal" }, + { label: "Notes", href: "/notes", type: "internal" }, + ]} versionId={versionId} pendingVersionId={pendingVersionId} onSelectVersion={handleSelectVersion} @@ -442,6 +447,13 @@ export default function App() { Paper {" "} ·{" "} + + Notes + {" "} + ·{" "} Expand - + {" "} + ·{" "} diff --git a/app/src/app/notes/[slug]/page.tsx b/app/src/app/notes/[slug]/page.tsx new file mode 100644 index 0000000..f541814 --- /dev/null +++ b/app/src/app/notes/[slug]/page.tsx @@ -0,0 +1,76 @@ +import type { Metadata } from "next"; +import { notFound } from "next/navigation"; + +import { NoteArticle, interpolateNoteText } from "../../../components/NotesContent"; +import SiteHeader from "../../../components/SiteHeader"; +import { getNote, notes } from "../../../notes"; + +export const dynamicParams = false; + +export function generateStaticParams() { + return notes.map((note) => ({ slug: note.slug })); +} + +export async function generateMetadata({ + params, +}: { + params: Promise<{ slug: string }>; +}): Promise { + const { slug } = await params; + const note = getNote(slug); + if (!note) notFound(); + const description = interpolateNoteText(note, note.paragraphs[0]); + const url = `https://policybench.org/notes/${note.slug}`; + return { + title: note.title, + description, + alternates: { canonical: `/notes/${note.slug}` }, + openGraph: { + type: "article", + url, + siteName: "PolicyBench", + title: note.title, + description, + publishedTime: note.date, + images: [ + { + url: "/og-image.png", + width: 1200, + height: 630, + alt: "PolicyBench — an LLM benchmark for tax and benefit calculation", + }, + ], + }, + twitter: { + card: "summary_large_image", + title: note.title, + description, + images: ["/og-image.png"], + }, + }; +} + +export default async function NotePage({ + params, +}: { + params: Promise<{ slug: string }>; +}) { + const { slug } = await params; + const note = getNote(slug); + if (!note) notFound(); + + return ( +
+ +
+ +
+
+ ); +} diff --git a/app/src/app/notes/page.tsx b/app/src/app/notes/page.tsx new file mode 100644 index 0000000..d72516a --- /dev/null +++ b/app/src/app/notes/page.tsx @@ -0,0 +1,63 @@ +import type { Metadata } from "next"; + +import { + NOTES_INTRO, + NotesPageContent, +} from "../../components/NotesContent"; +import SiteHeader from "../../components/SiteHeader"; + +export const metadata: Metadata = { + title: "Notes", + description: NOTES_INTRO, + alternates: { + canonical: "/notes", + }, + openGraph: { + type: "website", + url: "https://policybench.org/notes", + siteName: "PolicyBench", + title: "PolicyBench notes", + description: NOTES_INTRO, + images: [ + { + url: "/og-image.png", + width: 1200, + height: 630, + alt: "PolicyBench — an LLM benchmark for tax and benefit calculation", + }, + ], + }, + twitter: { + card: "summary_large_image", + title: "PolicyBench notes", + description: NOTES_INTRO, + images: ["/og-image.png"], + }, +}; + +export default function NotesPage() { + const expanded = ( + <> +

+ Notes +

+

+ {NOTES_INTRO} +

+ + ); + + return ( +
+ + +
+ ); +} diff --git a/app/src/app/paper/page.tsx b/app/src/app/paper/page.tsx index 550994c..9cc6b5a 100644 --- a/app/src/app/paper/page.tsx +++ b/app/src/app/paper/page.tsx @@ -69,11 +69,10 @@ export default function PaperPage() {

PolicyBench paper

diff --git a/app/src/app/sitemap.ts b/app/src/app/sitemap.ts index 816f7cc..6d26dbf 100644 --- a/app/src/app/sitemap.ts +++ b/app/src/app/sitemap.ts @@ -2,6 +2,7 @@ import type { MetadataRoute } from "next"; import rawData from "../data-summary.json"; import { listModels } from "../lib/modelPage"; +import { notes } from "../notes"; import type { DashboardBundle } from "../types"; export default function sitemap(): MetadataRoute.Sitemap { @@ -10,6 +11,11 @@ export default function sitemap(): MetadataRoute.Sitemap { changeFrequency: "monthly" as const, priority: 0.6, })); + const noteEntries = notes.map((note) => ({ + url: `https://policybench.org/notes/${note.slug}`, + changeFrequency: "monthly" as const, + priority: 0.5, + })); return [ { url: "https://policybench.org/", @@ -21,11 +27,17 @@ export default function sitemap(): MetadataRoute.Sitemap { changeFrequency: "monthly", priority: 0.7, }, + { + url: "https://policybench.org/notes", + changeFrequency: "weekly", + priority: 0.7, + }, { url: "https://policybench.org/expand", changeFrequency: "monthly", priority: 0.5, }, + ...noteEntries, ...modelEntries, ]; } diff --git a/app/src/components/Hero.tsx b/app/src/components/Hero.tsx index 48a64cc..8387420 100644 --- a/app/src/components/Hero.tsx +++ b/app/src/components/Hero.tsx @@ -1,6 +1,9 @@ import { DEFAULT_VERSION_ID } from "../lib/dataVersionsRuntime"; import type { BenchData, CountryCode } from "../types"; -import SiteHeader, { type HeaderNavItem } from "./SiteHeader"; +import SiteHeader, { + type HeaderActionLink, + type HeaderNavItem, +} from "./SiteHeader"; export default function Hero({ selectedView, @@ -9,6 +12,7 @@ export default function Hero({ availableViews, navItems, activeNav, + actionLinks, versionId, pendingVersionId, onSelectVersion, @@ -20,6 +24,7 @@ export default function Hero({ availableViews: CountryCode[]; navItems: readonly HeaderNavItem[]; activeNav: string; + actionLinks: readonly HeaderActionLink[]; versionId: string; pendingVersionId: string | null; onSelectVersion: (id: string) => void; @@ -56,7 +61,7 @@ export default function Hero({ versionId={versionId} pendingVersionId={pendingVersionId} onSelectVersion={onSelectVersion} - actionLink={{ label: "Paper", href: "/paper", type: "internal" }} + actionLinks={actionLinks} />
{ + const fact = note.facts[key]; + if (fact === undefined) throw new Error(`Unknown note fact: ${key}`); + return factText(key, fact); + }); +} + +function NoteLink({ href, children }: { href: string; children: ReactNode }) { + const className = "text-primary-strong underline-offset-2 hover:underline"; + if (href.startsWith("/")) { + return ( + + {children} + + ); + } + return ( + + {children} + + ); +} + +function factNode( + note: PolicyBenchNote, + key: string, + fact: NoteFact, +): ReactNode { + if (Array.isArray(fact)) { + return fact.map((item, index) => { + const link = note.data.find((entry) => entry.label === item); + return ( + + {index > 0 ? ", " : null} + {link ? {item} : item} + + ); + }); + } + if (key === "referenceAnnual" && typeof fact === "number") { + return ( + <> + {Math.round(fact)} + + + 1 + + + + ); + } + return factText(key, fact); +} + +function NoteParagraph({ + note, + paragraph, +}: { + note: PolicyBenchNote; + paragraph: string; +}) { + const content: ReactNode[] = []; + let cursor = 0; + for (const match of paragraph.matchAll(PLACEHOLDER)) { + const index = match.index; + if (index > cursor) content.push(paragraph.slice(cursor, index)); + const key = match[1]; + const fact = note.facts[key]; + if (fact === undefined) throw new Error(`Unknown note fact: ${key}`); + content.push({factNode(note, key, fact)}); + cursor = index + match[0].length; + } + if (cursor < paragraph.length) content.push(paragraph.slice(cursor)); + return

{content}

; +} + +export function NoteArticle({ + note, + titleLevel, + linkTitle = false, +}: { + note: PolicyBenchNote; + titleLevel: "h1" | "h2"; + linkTitle?: boolean; +}) { + const Title = titleLevel; + const title = linkTitle ? ( + + {note.title} + + ) : ( + note.title + ); + const annualReference = note.facts.referenceAnnual; + const DataTitle = titleLevel === "h1" ? "h2" : "h3"; + + return ( +
+
+ + + + Numbers from{" "} + + release {note.release} + {" "} + (board snapshot {note.boardSnapshot}) + +
+ + {title} + + +
+ {note.paragraphs.map((paragraph, index) => ( + + ))} +
+ + {typeof annualReference === "number" ? ( +

+ 1 Unrounded frozen reference: ${annualReference.toFixed(2)}. +

+ ) : null} + +
+ + Data + +
    + {note.data.map((entry) => ( +
  • + {entry.label} +
    • + ))} +
+
+
+ ); +} + +export function NotesPageContent() { + return ( +
+
+ {notes.map((note, index) => ( + + {index > 0 ? ( +
+ ) : null} + + + ))} +
+
+ ); +} diff --git a/app/src/components/SiteHeader.tsx b/app/src/components/SiteHeader.tsx index fe69491..00b4c43 100644 --- a/app/src/components/SiteHeader.tsx +++ b/app/src/components/SiteHeader.tsx @@ -151,6 +151,7 @@ export type SiteHeaderProps = { onSelectView?: (view: CountryCode) => void; availableViews?: CountryCode[]; actionLink?: HeaderActionLink; + actionLinks?: readonly HeaderActionLink[]; /** Active dataset-version id; when set with `onSelectVersion`, shows the * dataset selector in the header. */ versionId?: string; @@ -179,6 +180,7 @@ export default function SiteHeader({ onSelectView, availableViews, actionLink, + actionLinks, versionId, pendingVersionId, onSelectVersion, @@ -209,6 +211,10 @@ export default function SiteHeader({ availableViews && availableViews.length > 0 && selectedView && onSelectView; const showVersionSelector = versionId != null && onSelectVersion != null; const headerPositionClass = alwaysExpanded ? "relative z-40" : "sticky top-0 z-40"; + const resolvedActionLinks = [ + ...(actionLinks ?? []), + ...(actionLink ? [actionLink] : []), + ]; return (
@@ -304,25 +310,28 @@ export default function SiteHeader({
)} - {actionLink && ( -
- {actionLink.type === "external" ? ( + {resolvedActionLinks.map((link) => ( +
+ {link.type === "external" ? ( - {actionLink.label} + {link.label} ) : ( - {actionLink.label} + {link.label} )}
- )} + ))} ; + mentionRegexes?: Record; + data: NoteDataLink[]; + // The board snapshot and data release the facts were checked against; a + // note keeps its own release when later releases move the board. + boardSnapshot: string; + release: string; +}; + +export const notes = [ + gpt6AstraDebutsSecond, + sixSnapHouseholds, + claudeFable51Added, +] as PolicyBenchNote[]; + +export function getNote(slug: string): PolicyBenchNote | undefined { + return notes.find((note) => note.slug === slug); +} diff --git a/app/tests/notes.test.ts b/app/tests/notes.test.ts new file mode 100644 index 0000000..6f042fa --- /dev/null +++ b/app/tests/notes.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from "bun:test"; +import { readdirSync } from "node:fs"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; + +import { + NotesPageContent, + interpolateNoteText, +} from "../src/components/NotesContent"; +import { notes } from "../src/notes"; + +describe("notes", () => { + test("the index is newest first", () => { + expect(notes.map((note) => note.date)).toEqual( + [...notes].map((note) => note.date).sort().reverse(), + ); + }); + + test("slugs are unique and match JSON filenames", () => { + const slugs = notes.map((note) => note.slug); + expect(new Set(slugs).size).toBe(slugs.length); + const filenames = readdirSync(new URL("../src/notes", import.meta.url)) + .filter((name) => name.endsWith(".json")) + .map((name) => name.replace(/\.json$/, "")) + .sort(); + expect([...slugs].sort()).toEqual(filenames); + }); + + test("every paragraph placeholder resolves", () => { + for (const note of notes) { + const resolved = note.paragraphs.map((paragraph) => + interpolateNoteText(note, paragraph), + ); + expect(resolved.join(" ")).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*\}/); + } + }); + + test("the notes page renders every title without unresolved placeholders", () => { + const markup = renderToStaticMarkup(createElement(NotesPageContent)); + expect(markup).toContain("GPT-6 Astra debuts second: the two rules it invented"); + expect(markup).toContain("release dashboard-data-20260905c"); + // Whole-number board rates keep one decimal. + expect(markup).toContain("at 88.0% of answers within $1"); + expect(markup).not.toContain("at 88% of answers"); + expect(markup).toContain("release dashboard-data-20260901c"); + expect(markup).toContain("Six SNAP households the top three models deny"); + expect(markup).toContain("Claude Fable 5.1 added"); + expect(markup).not.toMatch(/\{[A-Za-z][A-Za-z0-9]*\}/); + }); +}); diff --git a/notes/data/astra_vs_sol_20260905.csv b/notes/data/astra_vs_sol_20260905.csv new file mode 100644 index 0000000..cb1f3f8 --- /dev/null +++ b/notes/data/astra_vs_sol_20260905.csv @@ -0,0 +1,63 @@ +scenario_id,variable,direction,cluster,prediction_astra,prediction_sol,reference,judge_failure_source,judge_failure_subtype,judge_annotation +scenario_008,spouse_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model invoked a ""modeled disability-based Medicare eligibility rule"" that does not exist — the eligibility test is age 65 or older, and the Spouse is 41. It never checked the 24-month SSDI-receipt condition that governs real under-65 Medicare entitlement, which is unsatisfiable here since no Social Security disability income is listed, so its 1 should be 0." +scenario_015,head_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model invoked ""PolicyEngine's disability-based Medicare eligibility treatment"" solely because the head is listed as disabled; Medicare eligibility here is determined by age against the 65 threshold, and the disability route to Medicare requires 24 months of SSDI entitlement rather than a disability indicator. With the head aged 36, employed at $45,000 in wages, and receiving no disability insurance benefits, the correct evaluation returns False." +scenario_027,spouse_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model invoked a ""PolicyEngine disability-based Medicare eligibility rule"" that does not exist in the model — is_medicare_eligible tests the individual's own age ≥ 65 — and offered no check on whether the spouse held any disability-insurance entitlement. The spouse is 39 with $0 disability income, so the age test returns False." +scenario_039,head_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model routed eligibility through ""the stated disability"" alone, skipping the entitlement condition that actually opens the under-65 pathway: 24 months of Social Security disability insurance benefits, ESRD, or ALS. All are unlisted and therefore zero, so only the age-65 test applies and age 61 fails it." +scenario_075,head_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model claimed eligibility under ""the modeled Medicare disability pathway,"" but no disability pathway is modeled — PolicyEngine determines Medicare eligibility from the age-65 threshold alone, returning False for a 62-year-old. Even the statutory pathway would fail here, since it requires 24 months of SSDI entitlement and the household's Social Security disability income is $0." +scenario_079,spouse_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model asserted a bare ""disability-based Medicare eligibility rule"" with no derivation, mapping the disabled flag directly to eligibility. The correct derivation requires age 65+ (spouse is 55) or 24 months of SSDI entitlement (spouse receives no Social Security disability income); the model's answer is consistent with reading the disability and blindness booleans as a standalone eligibility trigger, which is the SSI/SSDI disability definition, not a Medicare entitlement condition." +scenario_088,head_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model asserted eligibility ""under PolicyEngine's disability-based Medicare eligibility rule because the head is disabled,"" treating the disability flag as sufficient rather than as one input to the SSDI pathway that additionally requires 24 months of disability-benefit entitlement. With the head aged 56, $0 SSDI income, and no ESRD/ALS fact, no Medicare pathway opens and the value is No." +scenario_093,dependent1_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model claimed eligibility ""under the modeled disability-based Medicare eligibility criterion because disability is reported,"" attributing a disability criterion to PolicyEngine that is_medicare_eligible does not implement. The only modeled criterion is age >= 65, which Dependent 1 at age 27 fails, and the reported disability flag carries no SSDI entitlement, ESRD, or ALS status on these facts." +scenario_111,spouse_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"Cited a ""PolicyEngine disability-based Medicare eligibility rule"" fired by the is_disabled flag alone. Medicare's under-65 route requires 24 months of SSDI entitlement or an ESRD/ALS diagnosis, none of which is present in this household, so the age-65 threshold governs and the 49-year-old spouse is not eligible." +scenario_116,spouse_medicare_eligible,astra_only,disability_read_as_medicare,1.0,0.0,0.0,llm_error,age_disability,"The model asserted a 'modeled disability-based Medicare eligibility rule' that PolicyEngine does not have: is_medicare_eligible is keyed to the age-65 threshold, and the is_disabled flag drives SSI and the Medicaid aged/blind/disabled pathway, not Medicare. It mapped the disability boolean straight onto Medicare entitlement without requiring the SSDI receipt that the real 24-month pathway depends on, so it returned 1 for a 42-year-old with zero Social Security disability benefits." +scenario_037,federal_income_tax_before_refundable_credits,astra_only,esi_premium_netted_from_wages,952.1,1649.2,1649.2513427734375,llm_error,taxable_income_or_deductions,"It applied both the $16,100 standard deduction and the $9,629 overtime deduction correctly, but additionally subtracted the $5,789 employer-sponsored insurance premium from wages, which does not reduce the stated employment income, pushing taxable income down to $10,021 instead of $15,810.43. It also took a $50 saver's credit that is worth zero at AGI $41,539, above the 2026 single AGI ceiling for that credit." +scenario_037,payroll_tax,astra_only,esi_premium_netted_from_wages,2765.78,3208.64,3208.668701171875,llm_error,payroll_tax_base,"It correctly held that traditional 401(k) contributions and the FLSA overtime deduction leave payroll-tax wages untouched, then negated that by netting the $5,789 employer-sponsored insurance premium out of wages to reach a $36,154 base. The premium is not an employee section 125 salary reduction, so 7.65% applies to the full $41,943 for $3,208.67." +scenario_052,payroll_tax,astra_only,esi_premium_netted_from_wages,20558.11,20939.0,20939.0,llm_error,payroll_tax_base,"The model used the correct 2026 wage base ($184,500 × 6.2% = $11,439) but treated the $16,208 employer-plan premiums as pretax and computed Medicare and the Additional Medicare Tax on $483,792, giving $7,014.98 and $2,104.13. PolicyEngine applies both to the full $500,000 of gross wages, producing $7,250 and $2,250; that single base reduction accounts for the entire $380.89 shortfall." +scenario_064,payroll_tax,astra_only,esi_premium_netted_from_wages,5940.0,7443.07,7443.0673828125,llm_error,payroll_tax_base,"The model applied 7.65% to $77,647 after removing the $19,648 employer-sponsored insurance premium as a pre-tax deduction. Employee Social Security and Medicare are owed on the head's full $97,295 of gross wages, making the answer short by 7.65% of the premium it excluded." +scenario_089,federal_income_tax_before_refundable_credits,astra_only,esi_premium_netted_from_wages,5885.06,6820.0,6819.7119140625,llm_error,taxable_income_or_deductions,"It had the full structure right — $32,200 standard deduction, $3,467.43 overtime deduction, §199A — but additionally excluded the $7,789 employee share of employer-sponsored insurance premiums as 'eligible pretax insurance,' which does not reduce the $58,934.98 of employment income already net of the 401(k) deferral. That single extra $7,789 deduction at 12% is $934.65, exactly its shortfall against $6,819.71." +scenario_089,payroll_tax,astra_only,esi_premium_netted_from_wages,4444.04,5040.0,5039.91015625,llm_error,payroll_tax_base,"It computed 7.65% of $58,092 after removing the $7,789 ESI premium as a pretax salary reduction. It was right that there is no Additional Medicare Tax and no mandatory NC employee payroll tax, but the base is the full $65,881 of gross wages, giving $5,039.91." +scenario_089,state_income_tax_before_refundable_credits,astra_only,esi_premium_netted_from_wages,2909.14,3220.0,3219.935302734375,llm_error,taxable_income_or_deductions,"It applied the correct $25,500 joint standard deduction and 3.99% rate to an AGI of $98,410.90, which is the correct $106,200.13 less the $7,789 employer-sponsored insurance premium. That premium is not netted out of the stated gross wages, so taxable income is $80,700.13." +scenario_091,federal_income_tax_before_refundable_credits,astra_only,esi_premium_netted_from_wages,652.9,1350.0,1349.998779296875,llm_error,taxable_income_or_deductions,"It cut AGI to $29,594 by deducting the $6,589 of employer-sponsored insurance premiums, which lie outside the $27,000 of employment income and are not an above-the-line deduction; correct AGI is $35,012.56. That single error shrinks its ordinary base to $7,728 against the true $14,316.66, even though its $16,100 standard deduction, $627.91 overtime deduction, and $119.90 saver's credit were all right." +scenario_091,payroll_tax,astra_only,esi_premium_netted_from_wages,1561.44,2065.5,2065.5,llm_error,payroll_tax_base,"The model explicitly recharacterized the $6,589 employer-sponsored insurance figure as pre-tax employee premiums and applied 7.65% to the resulting $20,411. That input does not reduce Social Security and Medicare wages, which remain the stated $27,000, so the tax is $1,674.00 + $391.50 = $2,065.50; the model's other two calls (401(k) contributions stay FICA-taxable, Wisconsin adds no employee payroll tax) were correct." +scenario_093,payroll_tax,astra_only,esi_premium_netted_from_wages,10544.22,12469.5,12469.5,llm_error,payroll_tax_base,"The model applied 7.65% to $137,833 — gross wages less the three $8,389 employer-sponsored insurance premiums treated as pre-tax — for $10,544.22. Its handling of the wage ceiling, the Additional Medicare Tax, and the absence of a Missouri employee payroll tax was right; the single error is removing ESI premiums from a payroll tax base that PolicyEngine keeps at the full $163,000." +scenario_110,payroll_tax,astra_only,esi_premium_netted_from_wages,7122.99,7650.0,7650.0,llm_error,payroll_tax_base,"The model excluded the $6,889 employer-sponsored insurance premium as pretax and applied 7.65% to $93,111, correctly ruling out Additional Medicare Tax and Ohio employee payroll taxes but on the wrong base. The stated $100,000 of gross wages is the FICA base, yielding $6,200 Social Security and $1,450 Medicare for $7,650." +scenario_119,payroll_tax,astra_only,esi_premium_netted_from_wages,3718.74,4207.5,4207.5,llm_error,payroll_tax_base,"The model explicitly chose to treat the $6,389 employer-sponsored insurance premium as pretax and applied the 7.65% combined rate to $48,611. It got the two adjacent rules right — traditional 401(k) contributions do not reduce FICA wages and Virginia imposes no mandatory employee payroll tax — but the employer-paid ESI premium likewise leaves the base at $55,000, giving 7.65% x $55,000 = $4,207.50." +scenario_120,payroll_tax,astra_only,esi_premium_netted_from_wages,12890.69,13496.16,13496.1552734375,llm_error,payroll_tax_base,"The model assembled the right three levies — 6.2% Social Security, 1.45% Medicare, 0.5% CT paid leave — but applied them to ""estimated payroll-taxable wages of $158,168,"" removing the $7,429 employer-sponsored insurance premium from the $165,597 of wages. The base for all three is the full wage figure, making its answer 4.5% low across the board." +scenario_031,snap,astra_only,other,288.0,0.0,0.0,llm_error,categorical_eligibility,"Identified the decisive fact — no deductible shelter costs, leaving only the ~$207 standard deduction against $1,987.75/month of countable Social Security, pension, and IRA income — and then overrode it by paying the one-person minimum allotment on the strength of the 200%-of-poverty categorical-eligibility screen. California's MCE waives the gross income and asset tests but not the net income test, and net income of ~$1,780/month exceeds the ~$1,305 one-person limit, so the household is ineligible and the minimum benefit has nothing to floor." +scenario_057,state_refundable_credits,astra_only,other,23.24,33.2,33.20000076293945,llm_error,state_local_rule,"The model derived the federal EITC of $664 correctly but applied a 3.5% Louisiana match, the pre-2018 rate. Act 2 of the 2018 Second Extraordinary Session raised the credit to 5% of the federal EITC for tax years beginning on or after January 1, 2018, giving $33.20 instead of $23.24." +scenario_081,state_income_tax_before_refundable_credits,astra_only,other,8230.1,8238.7,8238.40625,llm_error,taxable_income_or_deductions,"It removed the $56 of taxable interest from the base by invoking the Massachusetts bank-interest exclusion, which does not reach this household's generic taxable interest, and offset the $110.12 dividend with the short-term capital loss. Both amounts are taxed at 5% — the interest in Part B and the dividend in Part A — so its otherwise correct 164,602 computation lands $8.31 short." +scenario_085,state_income_tax_before_refundable_credits,astra_only,other,51.42,0.0,0.0,llm_error,credit_phaseout,"The model reached the correct $51.42 gross tax but ruled out forgiveness because ""total household income precludes tax forgiveness,"" applying gross household income instead of Schedule SP eligibility income. Schedule SP excludes Social Security and Railroad Retirement benefits and old-age distributions from eligible retirement plans, so eligibility income is $1,675, forgiveness is 100%, and the liability is $0." +scenario_095,state_refundable_credits,astra_only,other,119.38,0.0,0.0,llm_error,state_local_rule,"Built the NJ EITC off a self-constructed ""age-adjusted federal credit"" instead of the federal EITC as computed; the state credit is 40% of the federal EITC amount, which is $0 here because the no-qualifying-child EITC is unavailable above age 64 and this filer is 73. It compounded the error by using self-employment income less half the self-employment tax as the earned income base — that subtraction belongs to AGI, not to EITC earned income — and its $119.38 is exactly 40% of a roughly $298 childless credit phased down against AGI of about $15,300, a credit the age ceiling eliminates entirely." +scenario_100,federal_refundable_credits,astra_only,other,822.4,2878.25,2878.0966796875,llm_error,taxable_income_or_deductions,"It based the EITC on $2,056 of wages after subtracting the $3,859 traditional 401(k) contribution, producing $822.40. The elective deferral lowers AGI but leaves the earned-income base at $5,915, so the EITC is $2,365.89 and earnings clear the $2,500 refundable-CTC threshold for a further $512.21." +scenario_101,federal_income_tax_before_refundable_credits,astra_only,other,13198.6,13418.6,13418.599609375,llm_error,taxable_income_or_deductions,"Its AGI, standard deduction, and 2026 brackets are all correct, but it subtracted an additional $1,000 auto-loan-interest deduction ($1,400 less the $400 phaseout for MAGI above $100,000) to reach $84,030 of taxable income instead of $85,030. The deduction is unavailable because the qualified-vehicle status flags are unlisted and therefore false, and 22% of the $1,000 is the entire $220 error." +scenario_118,state_refundable_credits,astra_only,other,75.0,375.0,375.0,llm_error,thresholds_rates,"The model found the right credit and confirmed property taxes were sufficient to reach the cap, then used the $75 ceiling that applies only when no household member is 65 or older. The head is 74, so §606(e) sets the maximum at $375, and that ceiling binds whether household gross income is the $2,800 of Social Security alone or the grossed-up figure the model assumed." +scenario_119,state_refundable_credits,astra_only,other,524.35,0.0,0.0,llm_error,state_local_rule,"Booked 20% of the federal EITC as a refundable Virginia credit, conflating the nonrefundable 20% credit with the separate 15% refundable election that expired for taxable years beginning on or after January 1, 2026. Its federal EITC input of $2,621.73 is also far above the phased-down two-child credit at ~$55,800 of AGI, compounding a rule error with a phaseout error." +scenario_123,payroll_tax,astra_only,other,11179.5,11194.0,11194.0,llm_error,state_local_rule,"It structured the answer correctly — $8,990 Social Security, $2,102.50 Medicare, plus a Pennsylvania employee unemployment contribution — but priced the PA piece at $87, which is 0.06% of $145,000, a superseded rate. The current Pennsylvania employee UC withholding rate is 0.07%, giving $101.50 and a total of $11,194." +scenario_045,federal_refundable_credits,sol_only,esi_premium_netted_from_wages,0.0,345.0,0.0,llm_error,taxable_income_or_deductions,"The model explicitly subtracted $21,208 of insurance premiums from gross wages as though they were pre-tax, yielding $15,068 of EITC income. Because that pre-tax treatment was not supplied, income remains $36,276 and the childless EITC is fully phased out." +scenario_045,payroll_tax,sol_only,esi_premium_netted_from_wages,2775.11,1153.0,2775.08251953125,llm_error,payroll_tax_base,"The model applied 7.65% to ""15068 in wages after pre-tax employer insurance premiums,"" deducting the whole $21,208 ESI premium from the FICA base. PolicyEngine taxes the full $36,276 of employment income, giving $2,249.09 of Social Security tax and $526.00 of Medicare tax for $2,775.08." +scenario_007,federal_income_tax_before_refundable_credits,sol_only,other,3707.7,3737.0,3707.695068359375,llm_error,taxable_income_or_deductions,"It added the $284 of tax-exempt private pension into provisional income; §86 adds back tax-exempt interest only, and the engine's provisional income excludes tax-exempt pension. That raised taxable benefits by $241.40 to $14,025.52 and taxable income to $33,205.52, producing $3,736.66, which it rounded to $3,737." +scenario_008,federal_refundable_credits,sol_only,other,12433.61,12476.0,12433.611328125,llm_error,taxable_income_or_deductions,"It named the $8,231 three-or-more-child EITC correctly but computed the earned-income-limited refundable CTC as $4,245, which is 15% x ($30,800 - $2,500) using gross self-employment income. Subtracting the deductible half of SE tax ($282.59) sets earned income at $30,517.41 and the refundable CTC at $4,202.61." +scenario_012,federal_refundable_credits,sol_only,other,6127.0,6227.0,6127.0,llm_error,thresholds_rates,"The model nailed the EITC leg at the correct $4,427 one-child maximum but applied an $1,800 per-child refundable CTC cap; the 2026 refundable ceiling is $1,700. The full $100 error is that single misremembered cap." +scenario_013,state_refundable_credits,sol_only,other,25.0,0.0,25.0,llm_error,state_local_rule,"The model required ""qualifying property tax, rent, dependents, or other credit-generating facts,"" importing the Form 140PTC and family income tax credit conditions. The increased excise tax credit imposes none of those and grants $25 to this resident single filer with $6,736 of federal AGI." +scenario_023,head_medicaid_eligible,sol_only,other,1.0,0.0,1.0,llm_error,categorical_eligibility,"It correctly computed MAGI of $22,535 above the expansion limit and correctly noted that disability alone does not create SSI-linked eligibility, but it never tested the pathway that disability plus earnings does create. The working-disabled buy-in category requires employment rather than SSI receipt and carries a 250% FPL income limit, which the head clears at 141% FPL." +scenario_026,child1_medicaid_eligible,sol_only,other,1.0,0.0,1.0,llm_error,state_local_rule,"It claimed MAGI exceeds North Carolina's limit for an 11-year-old without stating either figure. MAGI is $85,209 of wages less $2,908 of traditional 401(k)/IRA contributions = $82,301, which is 213% FPL for five people and under NC's ~216% FPL older-child limit." +scenario_026,child2_medicaid_eligible,sol_only,other,1.0,0.0,1.0,llm_error,thresholds_rates,"It asserted MAGI exceeds North Carolina's limit for an 11-year-old without computing either quantity. MAGI is $82,301 — wages less the traditional 401(k) and IRA contributions — which is 2.13x the 2026 five-person guideline and inside NC's 211%-plus older-child limit; the ineligible answer is consistent with applying the federal 133% floor." +scenario_026,child3_medicaid_eligible,sol_only,other,1.0,0.0,1.0,llm_error,thresholds_rates,"Asserted MAGI exceeds North Carolina's limit for a 9-year-old without stating either number. MAGI is $82,301 — 213% of the ~$38,600 five-person 2026 guideline — against the OLDER_CHILD limit of 211% FPL plus the 5-percentage-point disregard, so the answer tracks the 133% ACA minimum instead of NC's level." +scenario_026,federal_refundable_credits,sol_only,other,2013.4,2058.76,2013.4892578125,llm_error,taxable_income_or_deductions,"It reached $2,058.76 by deducting the $378 of auto loan interest in addition to the $32,200 standard deduction and the $7,746 overtime deduction, giving taxable income of $41,977 and tax of $4,541.24. The OBBBA vehicle-loan interest deduction requires a qualifying new, US-final-assembly vehicle financed after 2024 — conditions unlisted here and therefore false — so taxable income is $42,355, tax is $4,586.51, and the refundable remainder is $2,013.49, exactly $45.27 below its answer." +scenario_027,snap,sol_only,other,288.0,0.0,287.68316650390625,llm_error,categorical_eligibility,"It denied the household because SNAP net income sits above ""the applicable two-person limit,"" applying a net income test that Connecticut's broad-based categorical eligibility waives for this unit. Past that test, the $546 maximum allotment minus the $635.10 expected contribution is negative, so the minimum allotment of $23.84/$24.37 per month is paid." +scenario_028,state_refundable_credits,sol_only,other,62.41,0.0,62.41279220581055,llm_error,state_local_rule,"The model returned a one-line denial that any PA refundable individual income tax credit applies, performing no federal-EITC computation to feed the state match. Completing that step gives 10% of $624.13, or $62.41." +scenario_038,state_income_tax_before_refundable_credits,sol_only,other,0.0,282.0,0.0,llm_error,taxable_income_or_deductions,"The model applied Louisiana’s 3% rate to an implied positive base of about $9,400 after an unspecified married deduction. The applicable joint-filer and two-dependent calculation leaves zero taxable income, so the 3% rate has no positive base to tax." +scenario_044,state_refundable_credits,sol_only,other,0.0,250.0,0.0,llm_error,categorical_eligibility,"The model incorrectly placed the $250 Kansas food sales tax credit in the refundable-credit total. Kansas treats this credit as nonrefundable, leaving state_refundable_credits at $0." +scenario_045,head_medicaid_eligible,sol_only,other,0.0,1.0,0.0,llm_error,taxable_income_or_deductions,"The model's 'estimated MAGI of 15068' is the $36,276 wage net of the $21,208 employer-sponsored insurance premium, treating a health-coverage expense input as an income exclusion; Medicaid MAGI here is $36,276, or 2.27 x FPL, well above Michigan's 138% FPL expansion-adult limit of about $22,055. The expansion pathway it cited is therefore unavailable, and no non-MAGI pathway applies to a 44-year-old with no SSI and no disability." +scenario_062,federal_income_tax_before_refundable_credits,sol_only,other,0.0,4328.0,0.0,llm_error,taxable_income_or_deductions,"It listed survivor benefits as taxable income alongside 85%-included Social Security, producing AGI of about $63,245 and roughly $38,100 of taxable income after the standard, age, senior, and QBI deductions. The $28,800 survivor-benefits input does not enter countable income, which is $37,642; at a $26,986 provisional income only $993 of Social Security is taxable, giving AGI of $17,323 against ~$24,150 of deductions and $0 of tax." +scenario_064,child1_chip_eligible,sol_only,other,0.0,1.0,0.0,llm_error,health_coverage,"It correctly placed the child above Wisconsin's Medicaid ceiling for ages 6–18 but then asserted membership in the ""upper CHIP income band"" without computing MAGI against the poverty guideline. That band tops out at 306% FPL, and the tax unit's MAGI of about $130,700 — wages plus the $20,000 retirement distribution plus $33,350 of pass-through income, less the farm loss and the $3,000 capital-loss cap — is roughly 340% of the ~$38,400 five-person guideline, so the child overshoots the band rather than landing in it." +scenario_064,dependent2_chip_eligible,sol_only,other,0.0,1.0,0.0,llm_error,age_disability,"The model built its answer on a two-tier income band — 'above the Medicaid ceiling but within Wisconsin's upper CHIP income band' — after granting that dependent2 is 'under 19' and therefore inside the child pathway. PolicyEngine places age 18 above the CHIP child category's age ceiling, so dependent2 never enters the Medicaid-to-CHIP income band the model reasoned about; the trace shows Medicaid category NONE and is_chip_eligible False." +scenario_067,state_income_tax_before_refundable_credits,sol_only,other,2486.91,2442.66,2486.908935546875,llm_error,taxable_income_or_deductions,"Used the correct 2.95% 2026 rate but subtracted $4,500 of exemptions by adding a 'qualifying-child' amount for a 23-year-old. Indiana's additional $1,500 exemption is limited to a dependent child under 19, or under 24 and a full-time student, so total exemptions are $3,000 and the base is $84,302." +scenario_072,state_income_tax_before_refundable_credits,sol_only,other,0.0,356.0,0.0,llm_error,state_local_rule,"It applied only two estimated personal exemptions to the $20,166 remaining after excluding Social Security and veterans benefits, then taxed the balance at 4.25%. Michigan's senior standard deduction of $40,000 on a joint return where both filers have reached 67 applies to that same $20,166 of wage, dividend, and interest income and reduces Michigan taxable income to $0." +scenario_073,snap,sol_only,other,288.0,0.0,287.68316650390625,llm_error,thresholds_rates,"It concluded the income is too high to produce a positive allotment after deductions and the 30% benefit reduction. That computation matches the engine — $298 maximum minus a $526.50 expected contribution — but the result is floored at the one/two-person minimum allotment of 8% of the maximum, $23.84/mo, not zero." +scenario_079,state_refundable_credits,sol_only,other,50.0,0.0,50.0,llm_error,state_local_rule,"The model found no fact generating an Arizona refundable credit, overlooking that residency plus low AGI is the entire eligibility test for the increased excise tax credit. It pays $25 for each member of the tax unit up to $100, so this married couple receives $50." +scenario_095,state_income_tax_before_refundable_credits,sol_only,other,0.0,28.14,0.0,llm_error,state_local_rule,"The model applied the Social Security exclusion, the age-qualified retirement income exclusion for the $11,385 distribution, and the personal, age, and medical deductions, then ran the leftover self-employment income through the 1.4% bracket for $28.14. It omitted New Jersey's statutory zero-tax provision (N.J.S.A. 54A:2-1.1), under which NJ gross income of $4,198 — below the $20,000 surviving-spouse threshold and the $10,000 single threshold — produces no tax liability at all, so the value is $0.00." +scenario_098,snap,sol_only,other,0.0,3576.0,0.0,llm_error,taxable_income_or_deductions,"The model incorrectly treated the large rent and shelter deductions as reducing SNAP net income to zero. The household fails the income eligibility requirement before those deductions can support a benefit, so awarding the full maximum allotment was erroneous." +scenario_099,payroll_tax,sol_only,other,14767.5,14602.5,14767.5,llm_error,thresholds_rates,"Reproduced Social Security ($10,230) and Medicare ($2,392.50) exactly but estimated CA SDI at $1,980, i.e. the 2025 rate of 1.2%, rather than $2,145 at the 2026 rate of 1.3%. The entire $165 miss is the SDI rate parameter." +scenario_108,snap,sol_only,other,288.0,0.0,287.68316650390625,llm_error,categorical_eligibility,"It denied the benefit because countable net income stays above the one-person elderly-or-disabled net income limit, missing Wisconsin's broad-based categorical eligibility via a TANF-funded non-cash benefit, which screens gross income at 200% FPG — passed here at 1.98 times the $1,304.17 guideline — and waives that net income test and the asset test. The eligible household then receives the minimum allotment of $23.84/month, $24.37 from October, totaling $287.68." +scenario_109,federal_refundable_credits,sol_only,other,12038.09,12012.0,12038.0859375,llm_error,taxable_income_or_deductions,"The model paired the correct $8,231 EITC with an ACTC of $3,781, computed as 15% of net SE earnings ($30,000 × 0.9235 = $27,705) above $2,500. The correct earned-income base is gross SE income less half the SE tax, $27,880.57, giving $3,807.09 — the 92.35% haircut belongs to the SE-tax computation, not to the ACTC earned-income base." +scenario_121,head_medicare_eligible,sol_only,other,0.0,1.0,0.0,llm_error,age_disability,"It explicitly overrode the age test — ""eligible ... under the disability pathway even though the head is under age 65"" — asserting a pre-65 route that PolicyEngine does not implement for is_medicare_eligible. The disability_benefits input is not SSDI and carries no entitlement-duration information, leaving the age-65 comparison as the only operative condition, which the head fails at 53." diff --git a/notes/data/astra_vs_sol_20260905.csv.meta.json b/notes/data/astra_vs_sol_20260905.csv.meta.json new file mode 100644 index 0000000..9382695 --- /dev/null +++ b/notes/data/astra_vs_sol_20260905.csv.meta.json @@ -0,0 +1,8 @@ +{ + "release": "dashboard-data-20260905c", + "source_run": "us_full_run_20260612_policyengine_4_16_1_populace", + "annotations_sha256": "735b75d5a6599c8375a97f000b93c1904d6f27588480300bee0ab22430b893ef", + "rows": 62, + "generated_at_utc": "2026-09-05T16:36:43.512612+00:00", + "script": "scripts/astra_vs_sol_rows.py" +} diff --git a/notes/data/snap_pathways_20260901.csv b/notes/data/snap_pathways_20260901.csv new file mode 100644 index 0000000..eba7180 --- /dev/null +++ b/notes/data/snap_pathways_20260901.csv @@ -0,0 +1,101 @@ +scenario_id,state,household_size,snap_reference,snap_recomputed,meets_gross_income_test,meets_net_income_test,meets_asset_test,is_tanf_non_cash_eligible,snap_eligible,pathway +scenario_000,TX,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_001,VA,2,0.0,0.0,True,False,True,False,False,ineligible +scenario_002,WA,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_003,TX,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_004,NY,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_005,CA,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_007,ID,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_008,NJ,8,15246.9052734375,15246.9052734375,True,True,False,True,True,categorical_assets +scenario_009,NC,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_012,MS,3,4952.08935546875,4952.08935546875,True,True,True,False,True,ordinary +scenario_013,AZ,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_014,WV,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_015,IN,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_016,FL,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_018,AZ,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_020,TX,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_021,MO,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_022,CA,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_023,CA,1,461.3397216796875,461.3397216796875,False,True,True,True,True,categorical_income +scenario_025,OH,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_026,NC,5,0.0,0.0,False,False,False,False,False,ineligible +scenario_027,CT,2,287.68316650390625,287.68316650390625,True,False,False,True,True,categorical_both +scenario_028,PA,4,0.0,0.0,False,False,True,True,False,ineligible +scenario_029,OK,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_030,TX,1,287.68316650390625,287.68316650390625,False,False,True,True,True,categorical_income +scenario_031,CA,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_032,MN,3,0.0,0.0,False,False,False,True,False,ineligible +scenario_033,SD,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_036,NJ,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_037,NC,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_038,LA,4,7286.9443359375,7286.9443359375,True,True,True,True,True,ordinary +scenario_039,VA,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_040,AZ,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_042,WI,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_043,CO,1,3596.039794921875,3596.039794921875,True,True,True,True,True,ordinary +scenario_044,KS,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_045,MI,1,287.68316650390625,287.68316650390625,False,True,True,True,True,categorical_income +scenario_046,OK,4,0.0,0.0,False,False,False,False,False,ineligible +scenario_048,OH,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_049,NH,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_051,LA,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_052,TX,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_053,ID,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_054,NC,3,6125.68896484375,6125.68896484375,True,True,False,True,True,categorical_assets +scenario_055,FL,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_056,NJ,1,1140.0,1140.0,True,True,True,True,True,ordinary +scenario_057,LA,2,2669.217041015625,2669.217041015625,True,True,True,True,True,ordinary +scenario_059,FL,2,0.0,0.0,True,False,True,False,False,ineligible +scenario_060,TX,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_062,FL,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_064,WI,5,0.0,0.0,False,False,False,False,False,ineligible +scenario_066,VA,1,3596.039794921875,3596.039794921875,True,True,False,True,True,categorical_assets +scenario_067,IN,3,0.0,0.0,True,False,False,False,False,ineligible +scenario_068,MD,1,0.0,0.0,False,True,True,False,False,ineligible +scenario_070,IL,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_071,NY,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_072,MI,2,0.0,0.0,True,False,False,False,False,ineligible +scenario_073,MI,1,287.68316650390625,287.68316650390625,True,False,True,True,True,categorical_income +scenario_074,LA,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_075,FL,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_076,ID,3,0.0,0.0,False,False,False,False,False,ineligible +scenario_077,LA,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_078,MD,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_079,AZ,2,2428.017333984375,2428.017333984375,True,True,True,True,True,ordinary +scenario_080,PA,1,3596.039794921875,3596.039794921875,True,True,False,True,True,categorical_assets +scenario_081,MA,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_082,NY,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_083,TX,1,0.0,0.0,True,True,False,False,False,ineligible +scenario_084,NC,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_085,PA,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_086,GA,2,0.0,0.0,False,False,True,False,False,ineligible +scenario_088,TX,1,0.0,0.0,False,True,False,False,False,ineligible +scenario_089,NC,2,0.0,0.0,False,False,True,False,False,ineligible +scenario_090,KS,1,0.0,0.0,True,True,False,False,False,ineligible +scenario_091,WI,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_092,AL,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_093,MO,3,0.0,0.0,False,False,False,False,False,ineligible +scenario_095,NJ,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_098,IL,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_099,CA,4,0.0,0.0,False,False,False,False,False,ineligible +scenario_100,MT,3,8625.8896484375,8625.8896484375,True,True,True,True,True,ordinary +scenario_101,TX,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_102,NC,2,0.0,0.0,False,False,True,False,False,ineligible +scenario_104,NY,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_107,OH,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_108,WI,1,287.68316650390625,287.68316650390625,True,False,True,True,True,categorical_income +scenario_109,FL,5,8020.55419921875,8020.55419921875,True,True,True,True,True,ordinary +scenario_110,OH,1,0.0,0.0,False,False,False,False,False,ineligible +scenario_111,WA,2,0.0,0.0,True,False,True,False,False,ineligible +scenario_112,TX,1,287.68316650390625,287.68316650390625,True,True,True,True,True,ordinary +scenario_114,VA,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_115,AL,1,0.0,0.0,True,False,True,False,False,ineligible +scenario_116,FL,2,0.0,0.0,False,False,False,False,False,ineligible +scenario_117,AR,5,0.0,0.0,False,False,False,False,False,ineligible +scenario_118,NY,1,2903.9404296875,2903.9404296875,True,True,True,True,True,ordinary +scenario_119,VA,3,0.0,0.0,False,False,False,False,False,ineligible +scenario_120,CT,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_121,SC,1,0.0,0.0,False,False,True,False,False,ineligible +scenario_122,MN,1,0.0,0.0,True,False,False,False,False,ineligible +scenario_123,PA,3,0.0,0.0,False,False,False,False,False,ineligible diff --git a/notes/data/snap_pathways_20260901.csv.meta.json b/notes/data/snap_pathways_20260901.csv.meta.json new file mode 100644 index 0000000..a044852 --- /dev/null +++ b/notes/data/snap_pathways_20260901.csv.meta.json @@ -0,0 +1,6 @@ +{ + "policyengine_us_version": "1.723.0", + "generated_at_utc": "2026-09-03T11:52:27.778906+00:00", + "scenarios_sha256": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", + "script": "scripts/snap_pathways.py" +} diff --git a/scripts/astra_vs_sol_rows.py b/scripts/astra_vs_sol_rows.py new file mode 100644 index 0000000..fba95ba --- /dev/null +++ b/scripts/astra_vs_sol_rows.py @@ -0,0 +1,175 @@ +"""Row-level comparison of GPT-6 Astra and GPT-5.6 Sol on the frozen board. + +Writes ``notes/data/astra_vs_sol_20260905.csv``: every scored output one of +the two models gets right and the other misses, with both predictions, the +reference, the judge's row annotation, and a cluster label for Astra's two +repeated mechanisms. ``tests/test_notes.py`` regenerates the rows and +compares, so the note's counts stay tied to the frozen payload. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + +from policybench.snapshot_payload import read_run_payload + +ROOT = Path(__file__).resolve().parents[1] +RUN_DIR = ( + ROOT + / "paper/snapshot/20260501/runs" + / "us_full_run_20260612_policyengine_4_16_1_populace" +) +ANNOTATIONS = ( + ROOT + / "annotations/us_full_run_20260612_policyengine_4_16_1_populace" + / "us_audit_row_annotations.csv" +) +OUTPUT = ROOT / "notes/data/astra_vs_sol_20260905.csv" +SOL, ASTRA = "gpt-5.6-sol", "gpt-6-astra" + +# Astra's two repeated mechanisms, identified from the judge annotations: +# an employer-sponsored insurance premium netted out of wages before a tax +# base, and a disability flag read as Medicare eligibility before 65. +ESI_ROWS = { + ("scenario_037", "payroll_tax"), + ("scenario_052", "payroll_tax"), + ("scenario_064", "payroll_tax"), + ("scenario_089", "payroll_tax"), + ("scenario_091", "payroll_tax"), + ("scenario_093", "payroll_tax"), + ("scenario_110", "payroll_tax"), + ("scenario_119", "payroll_tax"), + ("scenario_120", "payroll_tax"), + ("scenario_037", "federal_income_tax_before_refundable_credits"), + ("scenario_089", "federal_income_tax_before_refundable_credits"), + ("scenario_091", "federal_income_tax_before_refundable_credits"), + ("scenario_089", "state_income_tax_before_refundable_credits"), +} +# Sol netted the same premium once, on scenario_045 (payroll tax and the +# refundable credits that key off earned income). +SOL_ESI_ROWS = { + ("scenario_045", "payroll_tax"), + ("scenario_045", "federal_refundable_credits"), +} +MEDICARE_SCENARIOS = { + "scenario_008", + "scenario_015", + "scenario_027", + "scenario_039", + "scenario_074", + "scenario_075", + "scenario_079", + "scenario_088", + "scenario_093", + "scenario_111", + "scenario_116", +} +CLUSTER_ESI = "esi_premium_netted_from_wages" +CLUSTER_MEDICARE = "disability_read_as_medicare" +CLUSTER_OTHER = "other" + +FIELDS = [ + "scenario_id", + "variable", + "direction", + "cluster", + "prediction_astra", + "prediction_sol", + "reference", + "judge_failure_source", + "judge_failure_subtype", + "judge_annotation", +] + + +def _hit(record: dict) -> bool: + return record.get("exact") == 100 + + +def cluster_for(scenario_id: str, variable: str, direction: str) -> str: + key = (scenario_id, variable) + if direction == "astra_only": + if key in ESI_ROWS: + return CLUSTER_ESI + if scenario_id in MEDICARE_SCENARIOS and "medicare" in variable: + return CLUSTER_MEDICARE + return CLUSTER_OTHER + return CLUSTER_ESI if key in SOL_ESI_ROWS else CLUSTER_OTHER + + +def comparison_rows(payload: dict, annotations: dict) -> list[dict]: + rows = [] + for scenario_id, variables in payload["scenarioPredictions"].items(): + for variable, models in variables.items(): + astra, sol = models.get(ASTRA), models.get(SOL) + if not astra or not sol or astra.get("scored") is False: + continue + astra_hit, sol_hit = _hit(astra), _hit(sol) + if astra_hit == sol_hit: + continue + direction = "astra_only" if sol_hit else "sol_only" + judged = annotations.get( + (ASTRA if direction == "astra_only" else SOL, scenario_id, variable), + {}, + ) + rows.append( + { + "scenario_id": scenario_id, + "variable": variable, + "direction": direction, + "cluster": cluster_for(scenario_id, variable, direction), + "prediction_astra": astra["prediction"], + "prediction_sol": sol["prediction"], + "reference": astra["groundTruth"], + "judge_failure_source": judged.get("failure_source", ""), + "judge_failure_subtype": judged.get("failure_subtype", ""), + "judge_annotation": judged.get("annotation", ""), + } + ) + rows.sort( + key=lambda row: ( + row["direction"], + row["cluster"], + row["scenario_id"], + row["variable"], + ) + ) + return rows + + +def load_annotations(path: Path = ANNOTATIONS) -> dict: + with path.open(encoding="utf-8", newline="") as source: + return { + (row["model"], row["scenario_id"], row["variable"]): row + for row in csv.DictReader(source) + } + + +def main() -> None: + payload = read_run_payload(RUN_DIR) + rows = comparison_rows(payload, load_annotations()) + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + with OUTPUT.open("w", encoding="utf-8", newline="") as sink: + writer = csv.DictWriter(sink, fieldnames=FIELDS) + writer.writeheader() + writer.writerows(rows) + meta = { + "release": "dashboard-data-20260905c", + "source_run": RUN_DIR.name, + "annotations_sha256": hashlib.sha256(ANNOTATIONS.read_bytes()).hexdigest(), + "rows": len(rows), + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "script": "scripts/astra_vs_sol_rows.py", + } + OUTPUT.with_suffix(OUTPUT.suffix + ".meta.json").write_text( + json.dumps(meta, indent=2) + "\n" + ) + print(f"wrote {OUTPUT.relative_to(ROOT)} ({len(rows)} rows)") + + +if __name__ == "__main__": + main() diff --git a/scripts/snap_pathways.py b/scripts/snap_pathways.py new file mode 100644 index 0000000..1fb1ac1 --- /dev/null +++ b/scripts/snap_pathways.py @@ -0,0 +1,239 @@ +"""Recompute SNAP eligibility pathways for the frozen 100-household snapshot.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from importlib.metadata import version +from pathlib import Path + +import numpy as np +import pandas as pd + +from policybench.ground_truth import ( + _build_us_vectorized_situation, + _extract_us_vectorized_value, +) +from policybench.policyengine_runtime import get_us_situation_simulation_class +from policybench.scenarios import load_scenarios_from_manifest + +ROOT = Path(__file__).resolve().parents[1] +RUN_DIR = ( + ROOT / "paper/snapshot/20260501/runs/" + "us_full_run_20260612_policyengine_4_16_1_populace" +) +SCENARIOS_PATH = RUN_DIR / "scenarios.csv" +REFERENCE_PATH = RUN_DIR / "reference_outputs.csv" +OUTPUT_PATH = ROOT / "notes/data/snap_pathways_20260901.csv" +META_PATH = OUTPUT_PATH.with_suffix(OUTPUT_PATH.suffix + ".meta.json") +SCRIPT_PATH = "scripts/snap_pathways.py" +YEAR = 2026 +MAX_REFERENCE_DIFFERENCE = 1.0 + +# policyengine-us 1.723.0 calls the underlying eligibility variable +# ``is_snap_eligible``. The output column keeps the requested shorter name and +# means a household has a positive computed allotment, matching how the note +# counts households that qualify in the frozen reference. The model-level gate +# is still calculated and checked while deriving each row. +VARIABLES = ( + "snap", + "meets_snap_gross_income_test", + "meets_snap_net_income_test", + "meets_snap_asset_test", + "is_tanf_non_cash_eligible", + "is_snap_eligible", +) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _bool(value: float) -> bool: + if value not in (0, 1): + raise ValueError(f"Expected a Boolean calculation, received {value!r}.") + return bool(value) + + +def _pathway( + *, + snap_eligible: bool, + gross: bool, + net: bool, + assets: bool, + categorical: bool, +) -> str: + if not snap_eligible: + return "ineligible" + if gross and net and assets: + return "ordinary" + if not categorical: + raise ValueError( + "A household with a positive SNAP allotment failed an ordinary " + "eligibility test without categorical eligibility." + ) + income_failure = not (gross and net) + if income_failure and not assets: + return "categorical_both" + if income_failure: + return "categorical_income" + if not assets: + return "categorical_assets" + raise AssertionError("Unreachable SNAP pathway combination.") + + +def _snap_references() -> pd.Series: + references = pd.read_csv(REFERENCE_PATH) + snap = references.loc[references["variable"].eq("snap")].copy() + if len(snap) != 100 or snap["scenario_id"].duplicated().any(): + raise ValueError( + "Expected exactly one frozen SNAP reference for each of 100 scenarios." + ) + return snap.set_index("scenario_id")["value"] + + +def build_rows() -> list[dict[str, object]]: + """Calculate pathway inputs with the same vectorized builder as references.""" + scenarios = load_scenarios_from_manifest(SCENARIOS_PATH) + if len(scenarios) != 100: + raise ValueError(f"Expected 100 frozen scenarios, found {len(scenarios)}.") + scenario_years = {scenario.year for scenario in scenarios} + if scenario_years != {YEAR}: + raise ValueError(f"Expected only {YEAR} scenarios, found {scenario_years}.") + + situation, scenario_indexes = _build_us_vectorized_situation(scenarios) + simulation_class = get_us_situation_simulation_class() + simulation = simulation_class(situation=situation) + + missing = [ + variable + for variable in VARIABLES + if variable not in simulation.tax_benefit_system.variables + ] + if missing: + raise ValueError(f"Installed policyengine-us is missing variables: {missing}") + + calculations: dict[str, tuple[np.ndarray, str]] = {} + for variable in VARIABLES: + entity_key = simulation.tax_benefit_system.variables[variable].entity.key + calculations[variable] = ( + np.asarray(simulation.calculate(variable, YEAR)), + entity_key, + ) + + references = _snap_references() + scenario_ids = {scenario.id for scenario in scenarios} + if scenario_ids != set(references.index): + raise ValueError("Frozen scenario and SNAP reference ids do not match.") + + rows: list[dict[str, object]] = [] + engine_eligible_without_allotment: list[str] = [] + for scenario in scenarios: + index = scenario_indexes[scenario.id] + + def result(variable: str) -> float: + values, entity_key = calculations[variable] + return _extract_us_vectorized_value( + values, + scenario=scenario, + variable=variable, + entity_key=entity_key, + index=index, + ) + + snap_reference = float(references.loc[scenario.id]) + snap_recomputed = result("snap") + if not np.isfinite([snap_reference, snap_recomputed]).all(): + raise ValueError(f"{scenario.id} has a non-finite SNAP value.") + gross = _bool(result("meets_snap_gross_income_test")) + net = _bool(result("meets_snap_net_income_test")) + assets = _bool(result("meets_snap_asset_test")) + categorical = _bool(result("is_tanf_non_cash_eligible")) + engine_eligible = _bool(result("is_snap_eligible")) + snap_eligible = snap_recomputed > 0 + if snap_eligible and not engine_eligible: + raise ValueError( + f"{scenario.id} has a positive SNAP allotment but fails " + "is_snap_eligible." + ) + if engine_eligible and not snap_eligible: + engine_eligible_without_allotment.append(scenario.id) + + rows.append( + { + "scenario_id": scenario.id, + "state": scenario.state, + "household_size": len(scenario.all_people), + "snap_reference": snap_reference, + "snap_recomputed": snap_recomputed, + "meets_gross_income_test": gross, + "meets_net_income_test": net, + "meets_asset_test": assets, + "is_tanf_non_cash_eligible": categorical, + "snap_eligible": snap_eligible, + "pathway": _pathway( + snap_eligible=snap_eligible, + gross=gross, + net=net, + assets=assets, + categorical=categorical, + ), + } + ) + + differences = [ + row + for row in rows + if abs(float(row["snap_recomputed"]) - float(row["snap_reference"])) + > MAX_REFERENCE_DIFFERENCE + ] + if differences: + details = "\n".join( + f" {row['scenario_id']}: reference={row['snap_reference']}, " + f"recomputed={row['snap_recomputed']}" + for row in differences + ) + raise RuntimeError( + "Recomputed SNAP differs from the frozen reference by more than " + f"${MAX_REFERENCE_DIFFERENCE:.0f}:\n{details}" + ) + + if engine_eligible_without_allotment: + print( + "PolicyEngine's is_snap_eligible gate is true but the computed " + "allotment is $0 for: " + ", ".join(engine_eligible_without_allotment) + ) + return rows + + +def main() -> None: + rows = build_rows() + OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame(rows).to_csv(OUTPUT_PATH, index=False) + metadata = { + "policyengine_us_version": version("policyengine-us"), + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "scenarios_sha256": _sha256(SCENARIOS_PATH), + "script": SCRIPT_PATH, + } + META_PATH.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + + pathway_counts = pd.Series(row["pathway"] for row in rows).value_counts() + print(f"Wrote {len(rows)} rows to {OUTPUT_PATH.relative_to(ROOT)}") + for pathway in ( + "ordinary", + "categorical_income", + "categorical_assets", + "categorical_both", + "ineligible", + ): + print(f" {pathway}: {int(pathway_counts.get(pathway, 0))}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_notes.py b/tests/test_notes.py new file mode 100644 index 0000000..e1c6b9a --- /dev/null +++ b/tests/test_notes.py @@ -0,0 +1,378 @@ +"""Checks that published note prose stays tied to committed evidence.""" + +from __future__ import annotations + +import csv +import gzip +import json +import re +import sys +from functools import cache +from pathlib import Path + +import pytest + +from policybench.snapshot_payload import read_run_payload + +ROOT = Path(__file__).resolve().parents[1] +NOTES_DIR = ROOT / "app/src/notes" +RUN_DIR = ( + ROOT / "paper/snapshot/20260501/runs/" + "us_full_run_20260612_policyengine_4_16_1_populace" +) +PREDICTIONS_PATH = RUN_DIR / "predictions.csv.gz" +REFERENCES_PATH = RUN_DIR / "reference_outputs.csv" +REFERENCE_META_PATH = RUN_DIR / "reference_outputs.csv.meta.json" +PATHWAYS_PATH = ROOT / "notes/data/snap_pathways_20260901.csv" +PATHWAYS_META_PATH = PATHWAYS_PATH.with_suffix(PATHWAYS_PATH.suffix + ".meta.json") +SENSITIVITY_PATH = ROOT / "sensitivity/data/claude-fable-5-1-thinking.json" +SENSITIVITY_NOTE_PATH = ROOT / "sensitivity/claude-thinking-2026-08.md" + +CLAUDE_NOTE = "2026-09-01-claude-fable-5-1-added" +SNAP_NOTE = "2026-09-03-six-snap-households" +ASTRA_NOTE = "2026-09-05-gpt-6-astra-debuts-second" +ASTRA_ROWS_PATH = ROOT / "notes/data/astra_vs_sol_20260905.csv" +TOP_MODELS = ("gpt-5.6-sol", "claude-fable-5.1", "kimi-k3") +PLACEHOLDER = re.compile(r"\{([A-Za-z][A-Za-z0-9]*)\}") + +csv.field_size_limit(sys.maxsize) + + +def _load_json(path: Path) -> dict: + with path.open(encoding="utf-8") as source: + return json.load(source) + + +def _note(slug: str) -> dict: + return _load_json(NOTES_DIR / f"{slug}.json") + + +@cache +def _dashboard() -> dict: + return read_run_payload(RUN_DIR) + + +@cache +def _frozen_release() -> str: + manifest = _load_json(ROOT / "paper/snapshot/20260501/manifest.json") + return manifest["published_dashboard_artifact"]["tag"] + + +# A note keeps the release its facts were checked against. Facts of a note on +# the frozen release are recomputed here; a note on a superseded release keeps +# the facts verified when that release was frozen (git history holds the run). +SUPERSEDED_RELEASES = {"dashboard-data-20260901c": "2026-09-01"} +CURRENT_RELEASE_SNAPSHOT = "2026-09-05" + + +@cache +def _snap_predictions() -> dict[str, list[dict[str, str]]]: + rows = {model: [] for model in TOP_MODELS} + with gzip.open(PREDICTIONS_PATH, "rt", encoding="utf-8", newline="") as source: + for row in csv.DictReader(source): + if row["variable"] == "snap" and row["model"] in rows: + rows[row["model"]].append(row) + assert all(len(model_rows) == 100 for model_rows in rows.values()) + return rows + + +def _snap_references() -> dict[str, float]: + with REFERENCES_PATH.open(encoding="utf-8", newline="") as source: + return { + row["scenario_id"]: float(row["value"]) + for row in csv.DictReader(source) + if row["variable"] == "snap" + } + + +def _display_one_decimal(value: float) -> float: + return float(f"{value:.1f}") + + +def _sensitivity_row(markdown: str, label: str) -> tuple[float, float]: + match = re.search( + rf"^\|\s*{re.escape(label)}\s*\|" + r"\s*([0-9.]+)\s*\([^)]*\)\s*\|" + r"\s*(?:\*\*)?([0-9.]+)(?:\*\*)?\s*\|", + markdown, + re.MULTILINE, + ) + assert match is not None + return float(match.group(1)), float(match.group(2)) + + +@pytest.mark.parametrize("path", sorted(NOTES_DIR.glob("*.json"))) +def test_note_schema_and_placeholders(path: Path) -> None: + note = _load_json(path) + assert note["slug"] == path.stem + assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", note["date"]) + if note["release"] == _frozen_release(): + assert note["boardSnapshot"] == CURRENT_RELEASE_SNAPSHOT + else: + assert note["boardSnapshot"] == SUPERSEDED_RELEASES[note["release"]] + assert note["boardSnapshot"] <= note["date"] or note["release"] != _frozen_release() + assert note["paragraphs"] + assert note["data"] + + placeholders = { + key + for paragraph in note["paragraphs"] + for key in PLACEHOLDER.findall(paragraph) + } + assert placeholders == set(note["facts"]) + + +def _recompute_against_frozen_snapshot(note: dict) -> bool: + """Whether the note's facts are recomputed here: only when its release is + the frozen snapshot's. A note on a superseded release keeps the facts that + were verified when that release was frozen; the test then checks the + release is one the repository has published.""" + if note["release"] == _frozen_release(): + return True + assert note["release"] in SUPERSEDED_RELEASES, note["release"] + return False + + +def test_claude_fable_note_facts() -> None: + note = _note(CLAUDE_NOTE) + if not _recompute_against_frozen_snapshot(note): + return + board_rows = [ + row for row in _dashboard()["modelStats"] if row["condition"] == "no_tools" + ] + target = next(row for row in board_rows if row["model"] == "claude-fable-5.1") + sensitivity = _load_json(SENSITIVITY_PATH) + sensitivity_exact = float(sensitivity["sensitivity"]["exact"]) + markdown = SENSITIVITY_NOTE_PATH.read_text(encoding="utf-8") + fable5_board, fable5_auto = _sensitivity_row(markdown, "Claude Fable 5") + + derived = { + "exactRate": _display_one_decimal(target["exact"]), + "rank": 1 + sum(row["exact"] > target["exact"] for row in board_rows), + "nModels": len(board_rows), + "parsed": target["nParsed"], + "answers": target["n"], + "autoRate": _display_one_decimal(sensitivity_exact), + "autoRank": 1 + sum(row["exact"] > sensitivity_exact for row in board_rows), + "fable5AutoRate": fable5_auto, + "fable5BoardRate": fable5_board, + "boardGap": _display_one_decimal(target["exact"] - fable5_board), + } + assert note["facts"] == derived + assert sensitivity["release"] == note["release"] + + +def _reference_monthly_minimum() -> float: + entries = _dashboard()["scenarioPredictions"]["scenario_030"]["snap"].values() + explanation = next(entry["referenceExplanation"] for entry in entries) + match = re.search(r"minimum allotment of \$([0-9.]+) per month", explanation) + assert match is not None + return float(match.group(1)) + + +def _mention_count(rows: list[dict[str, str]], pattern: str) -> int: + regex = re.compile(pattern, re.IGNORECASE) + return sum(bool(regex.search(row["explanation"] or "")) for row in rows) + + +def test_six_snap_households_note_facts() -> None: + note = _note(SNAP_NOTE) + if not _recompute_against_frozen_snapshot(note): + return + references = _snap_references() + eligible_ids = { + scenario_id for scenario_id, value in references.items() if value > 0 + } + predictions = _snap_predictions() + denied_by_model = { + model: { + row["scenario_id"] + for row in rows + if row["scenario_id"] in eligible_ids + and row["prediction"] + and float(row["prediction"]) == 0 + } + for model, rows in predictions.items() + } + denied_sets = list(denied_by_model.values()) + assert denied_sets[1:] == denied_sets[:-1] + denied_ids = sorted(denied_sets[0]) + denied_references = {references[scenario_id] for scenario_id in denied_ids} + assert len(denied_references) == 1 + + with PATHWAYS_PATH.open(encoding="utf-8", newline="") as source: + pathways = list(csv.DictReader(source)) + assert len(pathways) == 100 + assert all( + abs(float(row["snap_recomputed"]) - float(row["snap_reference"])) <= 1 + for row in pathways + ) + assert sum(row["snap_eligible"] == "True" for row in pathways) == len(eligible_ids) + categorical_rows = [ + row for row in pathways if row["pathway"].startswith("categorical_") + ] + categorical_income = [ + row + for row in pathways + if row["pathway"] in {"categorical_income", "categorical_both"} + ] + categorical_assets = [ + row for row in pathways if row["pathway"] == "categorical_assets" + ] + + pathway_meta = _load_json(PATHWAYS_META_PATH) + reference_meta = _load_json(REFERENCE_META_PATH) + regexes = note["mentionRegexes"] + derived = { + "eligibleCount": len(eligible_ids), + "deniedCount": len(denied_ids), + "deniedScenarios": denied_ids, + "referenceAnnual": round(next(iter(denied_references)), 2), + "referenceMonthly": _reference_monthly_minimum(), + "categoricalOnlyCount": len(categorical_rows), + "categoricalIncomeCount": len(categorical_income), + "categoricalAssetCount": len(categorical_assets), + "solCategoricalMentions": _mention_count( + predictions["gpt-5.6-sol"], regexes["categorical"] + ), + "solAssetMentions": _mention_count( + predictions["gpt-5.6-sol"], regexes["assets"] + ), + "fableBbceMentions": _mention_count( + predictions["claude-fable-5.1"], regexes["bbce"] + ), + "kimiCategoricalMentions": _mention_count( + predictions["kimi-k3"], regexes["categorical"] + ), + "pathwayEngineVersion": pathway_meta["policyengine_us_version"], + "referenceEngineVersion": reference_meta["policyengine_bundles"]["us"][ + "model_version" + ], + } + assert note["facts"] == derived + + +def test_categorical_asset_error_statements() -> None: + predictions = _snap_predictions() + with PATHWAYS_PATH.open(encoding="utf-8", newline="") as source: + asset_rows = { + row["scenario_id"]: float(row["snap_reference"]) + for row in csv.DictReader(source) + if row["pathway"] == "categorical_assets" + } + assert len(asset_rows) == 4 + + errors: dict[str, list[float]] = {} + for model in ("gpt-5.6-sol", "claude-fable-5.1"): + model_predictions = { + row["scenario_id"]: float(row["prediction"]) for row in predictions[model] + } + errors[model] = [ + abs(model_predictions[scenario_id] - reference) / reference + for scenario_id, reference in asset_rows.items() + ] + + sol_errors = errors["gpt-5.6-sol"] + fable_errors = errors["claude-fable-5.1"] + assert sum(error <= 0.01 for error in sol_errors) == 3 + assert sum(0.01 < error <= 0.10 for error in sol_errors) == 1 + assert sum(error <= 0.01 for error in fable_errors) == 4 + + +def _judge_annotations() -> dict[tuple[str, str, str], dict[str, str]]: + path = ( + ROOT + / "annotations/us_full_run_20260612_policyengine_4_16_1_populace" + / "us_audit_row_annotations.csv" + ) + with path.open(encoding="utf-8", newline="") as source: + return { + (row["model"], row["scenario_id"], row["variable"]): row + for row in csv.DictReader(source) + } + + +def test_astra_note_facts() -> None: + sys.path.insert(0, str(ROOT / "scripts")) + from astra_vs_sol_rows import ( + ASTRA, + CLUSTER_ESI, + CLUSTER_MEDICARE, + CLUSTER_OTHER, + SOL, + comparison_rows, + ) + + note = _note(ASTRA_NOTE) + assert note["release"] == _frozen_release() + payload = _dashboard() + annotations = _judge_annotations() + + expected_rows = comparison_rows(payload, annotations) + with ASTRA_ROWS_PATH.open(encoding="utf-8", newline="") as source: + committed = list(csv.DictReader(source)) + assert [ + (r["scenario_id"], r["variable"], r["direction"], r["cluster"]) + for r in committed + ] == [ + (r["scenario_id"], r["variable"], r["direction"], r["cluster"]) + for r in expected_rows + ] + + board_rows = [r for r in payload["modelStats"] if r["condition"] == "no_tools"] + by_model = {r["model"]: r for r in board_rows} + + def rank(model: str) -> int: + return 1 + sum(r["exact"] > by_model[model]["exact"] for r in board_rows) + + both_right = both_wrong = 0 + for variables in payload["scenarioPredictions"].values(): + for models in variables.values(): + astra, sol = models.get(ASTRA), models.get(SOL) + if not astra or not sol or astra.get("scored") is False: + continue + hits = (astra.get("exact") == 100, sol.get("exact") == 100) + both_right += hits == (True, True) + both_wrong += hits == (False, False) + astra_only = [r for r in expected_rows if r["direction"] == "astra_only"] + sol_only = [r for r in expected_rows if r["direction"] == "sol_only"] + excluded = { + (e["scenarioId"], e["variable"]) for e in payload["referenceExclusions"] + } + regex = re.compile(note["mentionRegexes"]["esi"], re.IGNORECASE) + + def mentions(model: str) -> int: + return sum( + bool(regex.search(row["annotation"] or "")) + for key, row in annotations.items() + if key[0] == model + ) + + derived = { + "astraExact": _display_one_decimal(by_model[ASTRA]["exact"]), + "astraRank": rank(ASTRA), + "nModels": len(board_rows), + "solExact": _display_one_decimal(by_model[SOL]["exact"]), + "fableExact": _display_one_decimal(by_model["claude-fable-5.1"]["exact"]), + "fableRank": rank("claude-fable-5.1"), + "scoredOutputs": by_model[ASTRA]["n"], + "totalOutputs": by_model[ASTRA]["n"] + len(excluded), + "excludedOutputs": len(excluded), + "bothRight": both_right, + "bothWrong": both_wrong, + "astraOnlyMisses": len(astra_only), + "solOnlyMisses": len(sol_only), + "esiRows": sum(r["cluster"] == CLUSTER_ESI for r in astra_only), + "medicareRows": sum(r["cluster"] == CLUSTER_MEDICARE for r in astra_only), + "otherRows": sum(r["cluster"] == CLUSTER_OTHER for r in astra_only), + "astraEsiMentions": mentions(ASTRA), + "solEsiMentions": mentions(SOL), + "solEsiRows": sum(r["cluster"] == CLUSTER_ESI for r in sol_only), + } + assert note["facts"] == derived + # The excluded Medicare row the note mentions is real and outside the rows. + assert ("scenario_074", "head_medicare_eligible") in excluded + assert not any( + r["scenario_id"] == "scenario_074" and "medicare" in r["variable"] + for r in expected_rows + )