report: aligned terminal tables; refuse a mismatched --final-repeat

README.md

@@ -310,7 +310,7 @@ at `REPRODUCE.md` instead.
 |---|---|
 | `/fairtask-eval show the evaluation set` | `npm run data:eval-set` — the 30-row table (instance, stratum, human scores, difficulty, sizes) and the per-stratum and per-repository counts. |
 | `/fairtask-eval score baseline v3-verify` | `npm run score -- baseline v3-verify` — decision accuracy, κ, TPR/TNR, missed/false alarms, per-axis agreement, cost and time, side by side. |
-| `/fairtask-eval the headline report, baseline versus final` | `node src/report.ts …` — the two tables in §4. |
+| `/fairtask-eval the headline report, baseline versus final` | `node src/report.ts …` — the two tables in §4, as aligned terminal tables. |
 | `/fairtask-eval audit evidence for v3-verify` | `npm run audit -- v3-verify` — share of cited quotes that do not exist where cited (needs `npm run data:workspaces`, which it tells you to run). |
 | `/fairtask-eval check the annotation provenance` | `npm run data:annotations -- --check` — SHA-256 of the committed file against the pinned source. |
 | `/fairtask-eval lay out astropy__astropy-12544` | `npm run show -- astropy__astropy-12544` — writes `examples/<id>/` and reads back the issue and the test patch. |
@@ -355,8 +355,9 @@ a complete command; rows with the same script differ by one flag.
 | `npm run score -- baseline v3-verify --detail` | Adds one row per instance: human scores, predicted scores, decision, cost. |
 | `npm run score -- baseline v3-verify --common` | Restricts every run to the instances all listed runs scored (for comparing partial runs). |
 | `npm run score -- baseline --json` | The summary as JSON instead of the table. |
-| `node src/report.ts --baseline baseline --final v3-verify` | Prints the README's headline comparison table for those two runs. |
-| `node src/report.ts --baseline baseline --final v5-cheap-probes --final-repeat v5-rerun` | Shows the final configuration as "first run · repeat run" in the headline table. |
+| `node src/report.ts --baseline baseline --final v3-verify` | Prints the headline comparison of those two runs as an aligned terminal table (the same rows as §4). |
+| `node src/report.ts --baseline baseline --final v3-verify --markdown` | The same as a Markdown pipe table — what `scripts/finalize-report.py` pastes into this README. |
+| `node src/report.ts --baseline baseline --final v5-cheap-probes --final-repeat v5-rerun` | Shows the final configuration as "first run · repeat run". The repeat must be a run of the same configuration; the script refuses otherwise. |
 | `node src/report.ts --baseline baseline --final v3-verify --runs baseline,v1-context,v3-verify` | Adds the all-systems table with those runs as rows. |
 | `npm run audit -- baseline v3-verify` | Runs the deterministic verifier over every stored verdict: how many cited quotes do not exist where cited. Needs cloned workspaces at the right commits (refuses otherwise). |
 | `npm run code-check` | The zero-LLM pre-check: identifiers used by graded tests, introduced by the gold patch, absent from issue and repository; prints per-instance hits and TPR/TNR against the human label. Needs workspaces. |

scripts/finalize-report.py

@@ -6,7 +6,7 @@
 final, runs = sys.argv[1], sys.argv[2:]
 final_repeat = next((r for r in runs if r == f"{final.split('-')[0]}-rerun"), None)
 subprocess.run(["node", "src/score.ts", *runs], cwd=root, check=True, capture_output=True)
-report = subprocess.run(["node", "src/report.ts", "--baseline", "baseline", "--final", final, *(["--final-repeat", final_repeat] if final_repeat else []), "--runs", ",".join(runs)], cwd=root, check=True, capture_output=True, text=True).stdout.strip()
+report = subprocess.run(["node", "src/report.ts", "--baseline", "baseline", "--final", final, *(["--final-repeat", final_repeat] if final_repeat else []), "--markdown", "--runs", ",".join(runs)], cwd=root, check=True, capture_output=True, text=True).stdout.strip()
 readme = (root / "README.md").read_text()
 start = readme.index("### Headline comparison"); end = readme.index("*How to read this.*")
 readme = readme[:start] + report + "\n\n" + readme[end:]

skills/fairtask-eval/SKILL.md

@@ -23,7 +23,7 @@ except `data` and `annotations`, which download pinned public files. None calls
    |---|---|
    | the evaluation set, the thirty cases, strata, which tasks | `npm run data:eval-set` (needs `data/raw/swebench_test.parquet`; if missing, run `curl -L -o data/raw/swebench_test.parquet https://huggingface.co/datasets/princeton-nlp/SWE-bench/resolve/main/data/test-00000-of-00001.parquet` first and say you did) |
    | score, accuracy, kappa, recall, a run id, compare runs | `npm run score -- <run ids…>` (default: `baseline v3-verify`; add `--detail` for per-instance rows) |
-   | the report, the headline table, baseline versus final | `node src/report.ts --baseline baseline --final v3-verify --final-repeat v5-rerun --runs <all run ids>` — the run ids are the directory names under `results/` |
+   | the report, the headline table, baseline versus final | `node src/report.ts --baseline baseline --final v3-verify --runs <all run ids>` — the run ids are the directory names under `results/`; the default (`v3-verify`) has no repeat run, so no `--final-repeat` (the script refuses a repeat of a different configuration). Prints aligned tables; `--markdown` gives the README's pipe tables |
    | evidence audit, bad evidence, fabricated quotes | `npm run audit -- <run ids…>` (needs cloned workspaces: `npm run data:workspaces` first, ~20 s) |
    | code check, novel identifiers, pre-check | `npm run code-check` (needs workspaces) |
    | provenance, annotations, checksum, data source | `npm run data:annotations -- --check` |

src/data/build-eval-set.ts

@@ -109,7 +109,7 @@ chosen.sort((a, b) => a.instance_id.localeCompare(b.instance_id));
 mkdirSync(`${ROOT}data/eval`, { recursive: true });
 writeFileSync(`${ROOT}data/eval/instances.json`, JSON.stringify(chosen, null, 2));
 console.log(`\nwrote ${chosen.length} instances -> data/eval/instances.json (seed ${SEED})`);
-console.log("stratum counts:", Object.fromEntries((Object.keys(QUOTA) as Stratum[]).map((s) => [s, chosen.filter((i) => i.stratum === s).length])));
-console.log("per repo:", Object.fromEntries(perRepo));
+console.log(`stratum counts: ${(Object.keys(QUOTA) as Stratum[]).map((s) => `${s} ${chosen.filter((i) => i.stratum === s).length}`).join(" · ")}`);
+console.log(`per repo: ${[...perRepo].map(([r, n]) => `${r} ${n}`).join(" · ")}`);
 console.log("\nid                                      strat          us fn diff              ps    patch  test");
 for (const i of chosen) console.log(`${i.instance_id.padEnd(40)}${i.stratum.padEnd(15)}${i.human.underspecified}  ${i.human.false_negative}  ${i.human.difficulty.padEnd(17)}${String(i.problem_statement.length).padStart(5)} ${String(i.patch.length).padStart(7)} ${String(i.test_patch.length).padStart(6)}${i.challenging ? "  <-- challenging" : ""}`);

src/lib/table.ts

@@ -0,0 +1,27 @@
+/**
+ * Render a header + rows either as an aligned terminal table (default: what an agent or a person sees in a
+ * terminal) or as a Markdown pipe table (what the README embeds). Terminal mode strips Markdown emphasis.
+ */
+export type TableFormat = "table" | "markdown";
+
+const plain = (s: string) => s.replace(/\*\*/g, "").replace(/`/g, "");
+// Display width: count code points, not UTF-16 units (κ, ▲, ▼, · are single-width).
+const width = (s: string) => Array.from(s).length;
+const pad = (s: string, w: number) => s + " ".repeat(Math.max(0, w - width(s)));
+
+export function renderTable(header: string[], rows: string[][], format: TableFormat = "table"): string {
+  if (format === "markdown") {
+    return [`| ${header.join(" | ")} |`, `|${header.map(() => "---").join("|")}|`, ...rows.map((r) => `| ${r.join(" | ")} |`)].join("\n");
+  }
+  const h = header.map(plain);
+  const rs = rows.map((r) => r.map(plain));
+  const widths = h.map((_, c) => Math.max(width(h[c]!), ...rs.map((r) => width(r[c] ?? ""))));
+  const line = (cells: string[]) => `│ ${cells.map((cell, c) => pad(cell, widths[c]!)).join(" │ ")} │`;
+  const rule = (l: string, m: string, r: string) => `${l}${widths.map((w) => "─".repeat(w + 2)).join(m)}${r}`;
+  return [rule("┌", "┬", "┐"), line(h), rule("├", "┼", "┤"), ...rs.map(line), rule("└", "┴", "┘")].join("\n");
+}
+
+/** A section title: Markdown heading in markdown mode, an underlined line in a terminal. */
+export function renderTitle(title: string, format: TableFormat = "table"): string {
+  return format === "markdown" ? `### ${title}\n` : `${title}\n${"═".repeat(width(title))}`;
+}

src/report.ts

@@ -1,12 +1,15 @@
 /**
  * Emit the README tables from results/<run>/summary.json (run `npm run score -- <runs...>` first).
  *
- * Usage: node src/report.ts --baseline baseline --final v3-verify --runs baseline,v1-context,... [--audit]
+ * Usage: node src/report.ts --baseline baseline --final v3-verify [--final-repeat <run>] [--runs baseline,v1-context,...] [--markdown]
+ *
+ * Prints aligned terminal tables by default; --markdown prints the pipe tables the README embeds.
  */
 import { existsSync, readFileSync } from "node:fs";
 import { resultsDir } from "./lib/paths.ts";
 import type { Summary } from "./score.ts";
 import { auditRun } from "./lib/audit.ts";
+import { renderTable, renderTitle, type TableFormat } from "./lib/table.ts";
 
 const args = new Map<string, string>();
 const argv = process.argv.slice(2);
@@ -20,33 +23,36 @@ const delta = (a: number, b: number, fmt: (x: number) => string, betterHigh = tr
 const base = load(args.get("baseline") ?? "baseline");
 const fin = load(args.get("final") ?? "v3-verify");
 const rep = args.get("final-repeat") ? load(args.get("final-repeat")!) : undefined;
+if (rep && rep.variant !== fin.variant) throw new Error(`--final-repeat ${rep.run_id} is a run of variant "${rep.variant}", not of the final configuration "${fin.variant}" (${fin.run_id}); a repeat must be the same configuration run again`);
+const format: TableFormat = args.get("markdown") ? "markdown" : "table";
 /** Show "first / repeat" when a repeat run of the final configuration exists. */
 const both = (f: (s: Summary) => string) => (rep ? `${f(fin)} · ${f(rep)}` : f(fin));
 const HUMAN_MIN_PER_TASK = 90; // assumption, see README
 const badRate = (id: string) => { const a = auditRun(id); return `${Math.round(100 * a.bad_items / (a.items || 1))}% (${a.bad_items}/${a.items})`; };
 
-console.log(`### Headline comparison (${base.n} cases, same cases and same deciding model for both${rep ? "; final shown as first run · repeat run" : ""})\n`);
-console.log(`| Metric | Simple baseline (\`${base.run_id}\`) | Agent solution (\`${fin.run_id}\`) | Change |`);
-console.log(`|---|---|---|---|`);
 const mean = (f: (s: Summary) => number) => (rep ? (f(fin) + f(rep)) / 2 : f(fin));
-const rng = (f: (s: Summary) => number, fmt: (x: number) => string) => (rep ? `${fmt(Math.min(f(fin), f(rep)))} to ${fmt(Math.max(f(fin), f(rep)))}` : fmt(f(fin)));
-console.log(`| **Primary: decision accuracy vs. human annotators** | ${pct(base.decision_accuracy)} | ${both((s) => pct(s.decision_accuracy))} | ${delta(base.decision_accuracy, mean((s) => s.decision_accuracy), (x) => `${Math.round(100 * x)} pts`)}${rep ? " (mean of 2 runs)" : ""} |`);
-console.log(`| Flag recall (human-flagged tasks caught) | ${pct(base.flag_recall)} | ${both((s) => pct(s.flag_recall))} | ${delta(base.flag_recall, mean((s) => s.flag_recall), (x) => `${Math.round(100 * x)} pts`)} |`);
-console.log(`| TPR / TNR (flagged caught / clean left alone; scored cases only) | ${pct(base.tpr)} (${Math.round(base.tpr * base.n_flag)}/${base.n_flag}) / ${pct(base.tnr)} (${Math.round(base.tnr * base.n_usable)}/${base.n_usable}) | ${both((s) => `${pct(s.tpr)} (${Math.round(s.tpr * s.n_flag)}/${s.n_flag}) / ${pct(s.tnr)} (${Math.round(s.tnr * s.n_usable)}/${s.n_usable})`)} | — |`);
-console.log(`| Missed problems / false alarms | ${base.missed_problems} / ${base.false_alarms} | ${both((s) => `${s.missed_problems} / ${s.false_alarms}`)} | — |`);
-console.log(`| Cohen's κ vs. humans | ${base.kappa.toFixed(2)} | ${both((s) => s.kappa.toFixed(2))} | ${delta(base.kappa, mean((s) => s.kappa), (x) => x.toFixed(2))} |`);
-console.log(`| Both axes flagged correctly | ${pct(base.both_axes_correct)} | ${both((s) => pct(s.both_axes_correct))} | ${delta(base.both_axes_correct, mean((s) => s.both_axes_correct), (x) => `${Math.round(100 * x)} pts`)} |`);
-void rng;
-console.log(`| Cited evidence that fails verification (item rate) | ${badRate(base.run_id)} | ${both((s) => badRate(s.run_id))} | — |`);
-console.log(`| Human time per task (assumption: ${HUMAN_MIN_PER_TASK} min of expert review today) | ${HUMAN_MIN_PER_TASK} min | reviewer checks cited evidence only | — |`);
-console.log(`| Machine wall-clock per task | ${Math.round(base.mean_duration_s)} s | ${Math.round(fin.mean_duration_s)} s | ${delta(base.mean_duration_s, fin.mean_duration_s, (x) => `${Math.round(x)} s`, false)} |`);
-console.log(`| Cost per task (USD, list price) | ${usd(base.mean_cost_usd)} | ${usd(fin.mean_cost_usd)} | ${delta(base.mean_cost_usd, fin.mean_cost_usd, usd, false)} |`);
-console.log(`| Challenging case (\`${fin.challenging?.instance_id}\`) | ${base.challenging?.correct ? "correct" : "wrong"} (${base.challenging?.pred}) | ${fin.challenging?.correct ? "correct" : "wrong"} (${fin.challenging?.pred}) | human: ${fin.challenging?.human} |`);
+const headline: string[][] = [
+  [`**Primary: decision accuracy vs. human annotators**`, `${pct(base.decision_accuracy)}`, `${both((s) => pct(s.decision_accuracy))}`, `${delta(base.decision_accuracy, mean((s) => s.decision_accuracy), (x) => `${Math.round(100 * x)} pts`)}${rep ? " (mean of 2 runs)" : ""}`],
+  [`Flag recall (human-flagged tasks caught)`, `${pct(base.flag_recall)}`, `${both((s) => pct(s.flag_recall))}`, `${delta(base.flag_recall, mean((s) => s.flag_recall), (x) => `${Math.round(100 * x)} pts`)}`],
+  [`TPR / TNR (flagged caught / clean left alone; scored cases only)`, `${pct(base.tpr)} (${Math.round(base.tpr * base.n_flag)}/${base.n_flag}) / ${pct(base.tnr)} (${Math.round(base.tnr * base.n_usable)}/${base.n_usable})`, `${both((s) => `${pct(s.tpr)} (${Math.round(s.tpr * s.n_flag)}/${s.n_flag}) / ${pct(s.tnr)} (${Math.round(s.tnr * s.n_usable)}/${s.n_usable})`)}`, `—`],
+  [`Missed problems / false alarms`, `${base.missed_problems} / ${base.false_alarms}`, `${both((s) => `${s.missed_problems} / ${s.false_alarms}`)}`, `—`],
+  [`Cohen's κ vs. humans`, `${base.kappa.toFixed(2)}`, `${both((s) => s.kappa.toFixed(2))}`, `${delta(base.kappa, mean((s) => s.kappa), (x) => x.toFixed(2))}`],
+  [`Both axes flagged correctly`, `${pct(base.both_axes_correct)}`, `${both((s) => pct(s.both_axes_correct))}`, `${delta(base.both_axes_correct, mean((s) => s.both_axes_correct), (x) => `${Math.round(100 * x)} pts`)}`],
+  [`Cited evidence that fails verification (item rate)`, `${badRate(base.run_id)}`, `${both((s) => badRate(s.run_id))}`, `—`],
+  [`Human time per task (assumption: ${HUMAN_MIN_PER_TASK} min of expert review today)`, `${HUMAN_MIN_PER_TASK} min`, `reviewer checks cited evidence only`, `—`],
+  [`Machine wall-clock per task`, `${Math.round(base.mean_duration_s)} s`, `${Math.round(fin.mean_duration_s)} s`, `${delta(base.mean_duration_s, fin.mean_duration_s, (x) => `${Math.round(x)} s`, false)}`],
+  [`Cost per task (USD, list price)`, `${usd(base.mean_cost_usd)}`, `${usd(fin.mean_cost_usd)}`, `${delta(base.mean_cost_usd, fin.mean_cost_usd, usd, false)}`],
+  [`Challenging case (\`${fin.challenging?.instance_id}\`)`, `${base.challenging?.correct ? "correct" : "wrong"} (${base.challenging?.pred})`, `${fin.challenging?.correct ? "correct" : "wrong"} (${fin.challenging?.pred})`, `human: ${fin.challenging?.human}`],
+];
+console.log(renderTitle(`Headline comparison (${base.n} cases, same cases and same deciding model for both${rep ? "; final shown as first run · repeat run" : ""})`, format));
+console.log(renderTable(["Metric", `Simple baseline (\`${base.run_id}\`)`, `Agent solution (\`${fin.run_id}\`)`, "Change"], headline, format));
 
 const runs = (args.get("runs") ?? "").split(",").filter(Boolean).map(load);
 if (runs.length) {
-  console.log(`\n### All systems on the same ${base.n} cases\n`);
-  console.log(`| Run | Decision acc. | κ | TPR / TNR | Missed / false alarms | Both axes | Bad evidence | Cost/task | Time/task |`);
-  console.log(`|---|---|---|---|---|---|---|---|---|`);
-  for (const r of runs) console.log(`| \`${r.run_id}\` | ${pct(r.decision_accuracy)} | ${r.kappa.toFixed(2)} | ${pct(r.tpr)} / ${pct(r.tnr)} | ${r.missed_problems} / ${r.false_alarms} | ${pct(r.both_axes_correct)} | ${badRate(r.run_id)} | ${usd(r.mean_cost_usd)} | ${Math.round(r.mean_duration_s)} s |`);
+  console.log("\n" + renderTitle(`All systems on the same ${base.n} cases`, format));
+  console.log(renderTable(
+    ["Run", "Decision acc.", "κ", "TPR / TNR", "Missed / false alarms", "Both axes", "Bad evidence", "Cost/task", "Time/task"],
+    runs.map((r) => [`\`${r.run_id}\``, pct(r.decision_accuracy), r.kappa.toFixed(2), `${pct(r.tpr)} / ${pct(r.tnr)}`, `${r.missed_problems} / ${r.false_alarms}`, pct(r.both_axes_correct), badRate(r.run_id), usd(r.mean_cost_usd), `${Math.round(r.mean_duration_s)} s`]),
+    format,
+  ));
 }

src/table.test.ts

@@ -0,0 +1,22 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { renderTable, renderTitle } from "./lib/table.ts";
+
+test("markdown mode emits a pipe table with a separator row", () => {
+  const md = renderTable(["A", "B"], [["**x**", "`y`"]], "markdown");
+  assert.equal(md, "| A | B |\n|---|---|\n| **x** | `y` |");
+});
+
+test("terminal mode strips emphasis and pads every column to the same width", () => {
+  const out = renderTable(["Metric", "κ"], [["**bold**", "0.29"], ["`run-id`", "▲ 1"]]);
+  const lines = out.split("\n");
+  assert.equal(lines.length, 6);
+  assert.ok(lines.every((l) => Array.from(l).length === Array.from(lines[0]!).length), "all rows share one display width");
+  assert.ok(!out.includes("**") && !out.includes("`"));
+  assert.ok(out.includes("│ bold   │ 0.29 │"), out);
+});
+
+test("titles switch between a heading and an underline", () => {
+  assert.equal(renderTitle("T", "markdown"), "### T\n");
+  assert.equal(renderTitle("Ti"), "Ti\n══");
+});