diff --git a/README.md b/README.md index 6924500..83be8b3 100644 --- a/README.md +++ b/README.md @@ -310,7 +310,7 @@ at `REPRODUCE.md` instead. |---|---| | `/fairtask-eval show the evaluation set` | `npm run data:eval-set` — the 30-row table (instance, stratum, human scores, difficulty, sizes) and the per-stratum and per-repository counts. | | `/fairtask-eval score baseline v3-verify` | `npm run score -- baseline v3-verify` — decision accuracy, κ, TPR/TNR, missed/false alarms, per-axis agreement, cost and time, side by side. | -| `/fairtask-eval the headline report, baseline versus final` | `node src/report.ts …` — the two tables in §4. | +| `/fairtask-eval the headline report, baseline versus final` | `node src/report.ts …` — the two tables in §4, as aligned terminal tables. | | `/fairtask-eval audit evidence for v3-verify` | `npm run audit -- v3-verify` — share of cited quotes that do not exist where cited (needs `npm run data:workspaces`, which it tells you to run). | | `/fairtask-eval check the annotation provenance` | `npm run data:annotations -- --check` — SHA-256 of the committed file against the pinned source. | | `/fairtask-eval lay out astropy__astropy-12544` | `npm run show -- astropy__astropy-12544` — writes `examples//` and reads back the issue and the test patch. | @@ -355,8 +355,9 @@ a complete command; rows with the same script differ by one flag. | `npm run score -- baseline v3-verify --detail` | Adds one row per instance: human scores, predicted scores, decision, cost. | | `npm run score -- baseline v3-verify --common` | Restricts every run to the instances all listed runs scored (for comparing partial runs). | | `npm run score -- baseline --json` | The summary as JSON instead of the table. | -| `node src/report.ts --baseline baseline --final v3-verify` | Prints the README's headline comparison table for those two runs. | -| `node src/report.ts --baseline baseline --final v5-cheap-probes --final-repeat v5-rerun` | Shows the final configuration as "first run · repeat run" in the headline table. | +| `node src/report.ts --baseline baseline --final v3-verify` | Prints the headline comparison of those two runs as an aligned terminal table (the same rows as §4). | +| `node src/report.ts --baseline baseline --final v3-verify --markdown` | The same as a Markdown pipe table — what `scripts/finalize-report.py` pastes into this README. | +| `node src/report.ts --baseline baseline --final v5-cheap-probes --final-repeat v5-rerun` | Shows the final configuration as "first run · repeat run". The repeat must be a run of the same configuration; the script refuses otherwise. | | `node src/report.ts --baseline baseline --final v3-verify --runs baseline,v1-context,v3-verify` | Adds the all-systems table with those runs as rows. | | `npm run audit -- baseline v3-verify` | Runs the deterministic verifier over every stored verdict: how many cited quotes do not exist where cited. Needs cloned workspaces at the right commits (refuses otherwise). | | `npm run code-check` | The zero-LLM pre-check: identifiers used by graded tests, introduced by the gold patch, absent from issue and repository; prints per-instance hits and TPR/TNR against the human label. Needs workspaces. | diff --git a/scripts/finalize-report.py b/scripts/finalize-report.py index fc5bf40..95a3b3f 100755 --- a/scripts/finalize-report.py +++ b/scripts/finalize-report.py @@ -6,7 +6,7 @@ final, runs = sys.argv[1], sys.argv[2:] final_repeat = next((r for r in runs if r == f"{final.split('-')[0]}-rerun"), None) subprocess.run(["node", "src/score.ts", *runs], cwd=root, check=True, capture_output=True) -report = subprocess.run(["node", "src/report.ts", "--baseline", "baseline", "--final", final, *(["--final-repeat", final_repeat] if final_repeat else []), "--runs", ",".join(runs)], cwd=root, check=True, capture_output=True, text=True).stdout.strip() +report = subprocess.run(["node", "src/report.ts", "--baseline", "baseline", "--final", final, *(["--final-repeat", final_repeat] if final_repeat else []), "--markdown", "--runs", ",".join(runs)], cwd=root, check=True, capture_output=True, text=True).stdout.strip() readme = (root / "README.md").read_text() start = readme.index("### Headline comparison"); end = readme.index("*How to read this.*") readme = readme[:start] + report + "\n\n" + readme[end:] diff --git a/skills/fairtask-eval/SKILL.md b/skills/fairtask-eval/SKILL.md index 35d90bc..bc5dfe4 100644 --- a/skills/fairtask-eval/SKILL.md +++ b/skills/fairtask-eval/SKILL.md @@ -23,7 +23,7 @@ except `data` and `annotations`, which download pinned public files. None calls |---|---| | the evaluation set, the thirty cases, strata, which tasks | `npm run data:eval-set` (needs `data/raw/swebench_test.parquet`; if missing, run `curl -L -o data/raw/swebench_test.parquet https://huggingface.co/datasets/princeton-nlp/SWE-bench/resolve/main/data/test-00000-of-00001.parquet` first and say you did) | | score, accuracy, kappa, recall, a run id, compare runs | `npm run score -- ` (default: `baseline v3-verify`; add `--detail` for per-instance rows) | - | the report, the headline table, baseline versus final | `node src/report.ts --baseline baseline --final v3-verify --final-repeat v5-rerun --runs ` — the run ids are the directory names under `results/` | + | the report, the headline table, baseline versus final | `node src/report.ts --baseline baseline --final v3-verify --runs ` — the run ids are the directory names under `results/`; the default (`v3-verify`) has no repeat run, so no `--final-repeat` (the script refuses a repeat of a different configuration). Prints aligned tables; `--markdown` gives the README's pipe tables | | evidence audit, bad evidence, fabricated quotes | `npm run audit -- ` (needs cloned workspaces: `npm run data:workspaces` first, ~20 s) | | code check, novel identifiers, pre-check | `npm run code-check` (needs workspaces) | | provenance, annotations, checksum, data source | `npm run data:annotations -- --check` | diff --git a/src/data/build-eval-set.ts b/src/data/build-eval-set.ts index a084554..9e5d35a 100644 --- a/src/data/build-eval-set.ts +++ b/src/data/build-eval-set.ts @@ -109,7 +109,7 @@ chosen.sort((a, b) => a.instance_id.localeCompare(b.instance_id)); mkdirSync(`${ROOT}data/eval`, { recursive: true }); writeFileSync(`${ROOT}data/eval/instances.json`, JSON.stringify(chosen, null, 2)); console.log(`\nwrote ${chosen.length} instances -> data/eval/instances.json (seed ${SEED})`); -console.log("stratum counts:", Object.fromEntries((Object.keys(QUOTA) as Stratum[]).map((s) => [s, chosen.filter((i) => i.stratum === s).length]))); -console.log("per repo:", Object.fromEntries(perRepo)); +console.log(`stratum counts: ${(Object.keys(QUOTA) as Stratum[]).map((s) => `${s} ${chosen.filter((i) => i.stratum === s).length}`).join(" · ")}`); +console.log(`per repo: ${[...perRepo].map(([r, n]) => `${r} ${n}`).join(" · ")}`); console.log("\nid strat us fn diff ps patch test"); for (const i of chosen) console.log(`${i.instance_id.padEnd(40)}${i.stratum.padEnd(15)}${i.human.underspecified} ${i.human.false_negative} ${i.human.difficulty.padEnd(17)}${String(i.problem_statement.length).padStart(5)} ${String(i.patch.length).padStart(7)} ${String(i.test_patch.length).padStart(6)}${i.challenging ? " <-- challenging" : ""}`); diff --git a/src/lib/table.ts b/src/lib/table.ts new file mode 100644 index 0000000..65c149e --- /dev/null +++ b/src/lib/table.ts @@ -0,0 +1,27 @@ +/** + * Render a header + rows either as an aligned terminal table (default: what an agent or a person sees in a + * terminal) or as a Markdown pipe table (what the README embeds). Terminal mode strips Markdown emphasis. + */ +export type TableFormat = "table" | "markdown"; + +const plain = (s: string) => s.replace(/\*\*/g, "").replace(/`/g, ""); +// Display width: count code points, not UTF-16 units (κ, ▲, ▼, · are single-width). +const width = (s: string) => Array.from(s).length; +const pad = (s: string, w: number) => s + " ".repeat(Math.max(0, w - width(s))); + +export function renderTable(header: string[], rows: string[][], format: TableFormat = "table"): string { + if (format === "markdown") { + return [`| ${header.join(" | ")} |`, `|${header.map(() => "---").join("|")}|`, ...rows.map((r) => `| ${r.join(" | ")} |`)].join("\n"); + } + const h = header.map(plain); + const rs = rows.map((r) => r.map(plain)); + const widths = h.map((_, c) => Math.max(width(h[c]!), ...rs.map((r) => width(r[c] ?? "")))); + const line = (cells: string[]) => `│ ${cells.map((cell, c) => pad(cell, widths[c]!)).join(" │ ")} │`; + const rule = (l: string, m: string, r: string) => `${l}${widths.map((w) => "─".repeat(w + 2)).join(m)}${r}`; + return [rule("┌", "┬", "┐"), line(h), rule("├", "┼", "┤"), ...rs.map(line), rule("└", "┴", "┘")].join("\n"); +} + +/** A section title: Markdown heading in markdown mode, an underlined line in a terminal. */ +export function renderTitle(title: string, format: TableFormat = "table"): string { + return format === "markdown" ? `### ${title}\n` : `${title}\n${"═".repeat(width(title))}`; +} diff --git a/src/report.ts b/src/report.ts index dd374ad..b49673a 100644 --- a/src/report.ts +++ b/src/report.ts @@ -1,12 +1,15 @@ /** * Emit the README tables from results//summary.json (run `npm run score -- ` first). * - * Usage: node src/report.ts --baseline baseline --final v3-verify --runs baseline,v1-context,... [--audit] + * Usage: node src/report.ts --baseline baseline --final v3-verify [--final-repeat ] [--runs baseline,v1-context,...] [--markdown] + * + * Prints aligned terminal tables by default; --markdown prints the pipe tables the README embeds. */ import { existsSync, readFileSync } from "node:fs"; import { resultsDir } from "./lib/paths.ts"; import type { Summary } from "./score.ts"; import { auditRun } from "./lib/audit.ts"; +import { renderTable, renderTitle, type TableFormat } from "./lib/table.ts"; const args = new Map(); const argv = process.argv.slice(2); @@ -20,33 +23,36 @@ const delta = (a: number, b: number, fmt: (x: number) => string, betterHigh = tr const base = load(args.get("baseline") ?? "baseline"); const fin = load(args.get("final") ?? "v3-verify"); const rep = args.get("final-repeat") ? load(args.get("final-repeat")!) : undefined; +if (rep && rep.variant !== fin.variant) throw new Error(`--final-repeat ${rep.run_id} is a run of variant "${rep.variant}", not of the final configuration "${fin.variant}" (${fin.run_id}); a repeat must be the same configuration run again`); +const format: TableFormat = args.get("markdown") ? "markdown" : "table"; /** Show "first / repeat" when a repeat run of the final configuration exists. */ const both = (f: (s: Summary) => string) => (rep ? `${f(fin)} · ${f(rep)}` : f(fin)); const HUMAN_MIN_PER_TASK = 90; // assumption, see README const badRate = (id: string) => { const a = auditRun(id); return `${Math.round(100 * a.bad_items / (a.items || 1))}% (${a.bad_items}/${a.items})`; }; -console.log(`### Headline comparison (${base.n} cases, same cases and same deciding model for both${rep ? "; final shown as first run · repeat run" : ""})\n`); -console.log(`| Metric | Simple baseline (\`${base.run_id}\`) | Agent solution (\`${fin.run_id}\`) | Change |`); -console.log(`|---|---|---|---|`); const mean = (f: (s: Summary) => number) => (rep ? (f(fin) + f(rep)) / 2 : f(fin)); -const rng = (f: (s: Summary) => number, fmt: (x: number) => string) => (rep ? `${fmt(Math.min(f(fin), f(rep)))} to ${fmt(Math.max(f(fin), f(rep)))}` : fmt(f(fin))); -console.log(`| **Primary: decision accuracy vs. human annotators** | ${pct(base.decision_accuracy)} | ${both((s) => pct(s.decision_accuracy))} | ${delta(base.decision_accuracy, mean((s) => s.decision_accuracy), (x) => `${Math.round(100 * x)} pts`)}${rep ? " (mean of 2 runs)" : ""} |`); -console.log(`| Flag recall (human-flagged tasks caught) | ${pct(base.flag_recall)} | ${both((s) => pct(s.flag_recall))} | ${delta(base.flag_recall, mean((s) => s.flag_recall), (x) => `${Math.round(100 * x)} pts`)} |`); -console.log(`| TPR / TNR (flagged caught / clean left alone; scored cases only) | ${pct(base.tpr)} (${Math.round(base.tpr * base.n_flag)}/${base.n_flag}) / ${pct(base.tnr)} (${Math.round(base.tnr * base.n_usable)}/${base.n_usable}) | ${both((s) => `${pct(s.tpr)} (${Math.round(s.tpr * s.n_flag)}/${s.n_flag}) / ${pct(s.tnr)} (${Math.round(s.tnr * s.n_usable)}/${s.n_usable})`)} | — |`); -console.log(`| Missed problems / false alarms | ${base.missed_problems} / ${base.false_alarms} | ${both((s) => `${s.missed_problems} / ${s.false_alarms}`)} | — |`); -console.log(`| Cohen's κ vs. humans | ${base.kappa.toFixed(2)} | ${both((s) => s.kappa.toFixed(2))} | ${delta(base.kappa, mean((s) => s.kappa), (x) => x.toFixed(2))} |`); -console.log(`| Both axes flagged correctly | ${pct(base.both_axes_correct)} | ${both((s) => pct(s.both_axes_correct))} | ${delta(base.both_axes_correct, mean((s) => s.both_axes_correct), (x) => `${Math.round(100 * x)} pts`)} |`); -void rng; -console.log(`| Cited evidence that fails verification (item rate) | ${badRate(base.run_id)} | ${both((s) => badRate(s.run_id))} | — |`); -console.log(`| Human time per task (assumption: ${HUMAN_MIN_PER_TASK} min of expert review today) | ${HUMAN_MIN_PER_TASK} min | reviewer checks cited evidence only | — |`); -console.log(`| Machine wall-clock per task | ${Math.round(base.mean_duration_s)} s | ${Math.round(fin.mean_duration_s)} s | ${delta(base.mean_duration_s, fin.mean_duration_s, (x) => `${Math.round(x)} s`, false)} |`); -console.log(`| Cost per task (USD, list price) | ${usd(base.mean_cost_usd)} | ${usd(fin.mean_cost_usd)} | ${delta(base.mean_cost_usd, fin.mean_cost_usd, usd, false)} |`); -console.log(`| Challenging case (\`${fin.challenging?.instance_id}\`) | ${base.challenging?.correct ? "correct" : "wrong"} (${base.challenging?.pred}) | ${fin.challenging?.correct ? "correct" : "wrong"} (${fin.challenging?.pred}) | human: ${fin.challenging?.human} |`); +const headline: string[][] = [ + [`**Primary: decision accuracy vs. human annotators**`, `${pct(base.decision_accuracy)}`, `${both((s) => pct(s.decision_accuracy))}`, `${delta(base.decision_accuracy, mean((s) => s.decision_accuracy), (x) => `${Math.round(100 * x)} pts`)}${rep ? " (mean of 2 runs)" : ""}`], + [`Flag recall (human-flagged tasks caught)`, `${pct(base.flag_recall)}`, `${both((s) => pct(s.flag_recall))}`, `${delta(base.flag_recall, mean((s) => s.flag_recall), (x) => `${Math.round(100 * x)} pts`)}`], + [`TPR / TNR (flagged caught / clean left alone; scored cases only)`, `${pct(base.tpr)} (${Math.round(base.tpr * base.n_flag)}/${base.n_flag}) / ${pct(base.tnr)} (${Math.round(base.tnr * base.n_usable)}/${base.n_usable})`, `${both((s) => `${pct(s.tpr)} (${Math.round(s.tpr * s.n_flag)}/${s.n_flag}) / ${pct(s.tnr)} (${Math.round(s.tnr * s.n_usable)}/${s.n_usable})`)}`, `—`], + [`Missed problems / false alarms`, `${base.missed_problems} / ${base.false_alarms}`, `${both((s) => `${s.missed_problems} / ${s.false_alarms}`)}`, `—`], + [`Cohen's κ vs. humans`, `${base.kappa.toFixed(2)}`, `${both((s) => s.kappa.toFixed(2))}`, `${delta(base.kappa, mean((s) => s.kappa), (x) => x.toFixed(2))}`], + [`Both axes flagged correctly`, `${pct(base.both_axes_correct)}`, `${both((s) => pct(s.both_axes_correct))}`, `${delta(base.both_axes_correct, mean((s) => s.both_axes_correct), (x) => `${Math.round(100 * x)} pts`)}`], + [`Cited evidence that fails verification (item rate)`, `${badRate(base.run_id)}`, `${both((s) => badRate(s.run_id))}`, `—`], + [`Human time per task (assumption: ${HUMAN_MIN_PER_TASK} min of expert review today)`, `${HUMAN_MIN_PER_TASK} min`, `reviewer checks cited evidence only`, `—`], + [`Machine wall-clock per task`, `${Math.round(base.mean_duration_s)} s`, `${Math.round(fin.mean_duration_s)} s`, `${delta(base.mean_duration_s, fin.mean_duration_s, (x) => `${Math.round(x)} s`, false)}`], + [`Cost per task (USD, list price)`, `${usd(base.mean_cost_usd)}`, `${usd(fin.mean_cost_usd)}`, `${delta(base.mean_cost_usd, fin.mean_cost_usd, usd, false)}`], + [`Challenging case (\`${fin.challenging?.instance_id}\`)`, `${base.challenging?.correct ? "correct" : "wrong"} (${base.challenging?.pred})`, `${fin.challenging?.correct ? "correct" : "wrong"} (${fin.challenging?.pred})`, `human: ${fin.challenging?.human}`], +]; +console.log(renderTitle(`Headline comparison (${base.n} cases, same cases and same deciding model for both${rep ? "; final shown as first run · repeat run" : ""})`, format)); +console.log(renderTable(["Metric", `Simple baseline (\`${base.run_id}\`)`, `Agent solution (\`${fin.run_id}\`)`, "Change"], headline, format)); const runs = (args.get("runs") ?? "").split(",").filter(Boolean).map(load); if (runs.length) { - console.log(`\n### All systems on the same ${base.n} cases\n`); - console.log(`| Run | Decision acc. | κ | TPR / TNR | Missed / false alarms | Both axes | Bad evidence | Cost/task | Time/task |`); - console.log(`|---|---|---|---|---|---|---|---|---|`); - for (const r of runs) console.log(`| \`${r.run_id}\` | ${pct(r.decision_accuracy)} | ${r.kappa.toFixed(2)} | ${pct(r.tpr)} / ${pct(r.tnr)} | ${r.missed_problems} / ${r.false_alarms} | ${pct(r.both_axes_correct)} | ${badRate(r.run_id)} | ${usd(r.mean_cost_usd)} | ${Math.round(r.mean_duration_s)} s |`); + console.log("\n" + renderTitle(`All systems on the same ${base.n} cases`, format)); + console.log(renderTable( + ["Run", "Decision acc.", "κ", "TPR / TNR", "Missed / false alarms", "Both axes", "Bad evidence", "Cost/task", "Time/task"], + runs.map((r) => [`\`${r.run_id}\``, pct(r.decision_accuracy), r.kappa.toFixed(2), `${pct(r.tpr)} / ${pct(r.tnr)}`, `${r.missed_problems} / ${r.false_alarms}`, pct(r.both_axes_correct), badRate(r.run_id), usd(r.mean_cost_usd), `${Math.round(r.mean_duration_s)} s`]), + format, + )); } diff --git a/src/table.test.ts b/src/table.test.ts new file mode 100644 index 0000000..21203c5 --- /dev/null +++ b/src/table.test.ts @@ -0,0 +1,22 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { renderTable, renderTitle } from "./lib/table.ts"; + +test("markdown mode emits a pipe table with a separator row", () => { + const md = renderTable(["A", "B"], [["**x**", "`y`"]], "markdown"); + assert.equal(md, "| A | B |\n|---|---|\n| **x** | `y` |"); +}); + +test("terminal mode strips emphasis and pads every column to the same width", () => { + const out = renderTable(["Metric", "κ"], [["**bold**", "0.29"], ["`run-id`", "▲ 1"]]); + const lines = out.split("\n"); + assert.equal(lines.length, 6); + assert.ok(lines.every((l) => Array.from(l).length === Array.from(lines[0]!).length), "all rows share one display width"); + assert.ok(!out.includes("**") && !out.includes("`")); + assert.ok(out.includes("│ bold │ 0.29 │"), out); +}); + +test("titles switch between a heading and an underline", () => { + assert.equal(renderTitle("T", "markdown"), "### T\n"); + assert.equal(renderTitle("Ti"), "Ti\n══"); +});