From b7901dc181f864d9ed5eab9fb2bcf2fdb30509d7 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 10 Sep 2026 01:13:02 -0700 Subject: [PATCH 1/3] feat(bench): add the public audit benchmark Step 0 of the traces audit hill-climb: a benchmark that says whether an audit arm beats a fleet of general subagents on a coding-agent session, instead of asserting it. - bench/audit/fixtures.ts generates three synthetic sessions (a Codex operator session over 600 spans, its forked child, and a Claude Code session with Task subagents) and records each planted fact as it writes the record that carries it, so the gold never passes through a traces adapter. - bench/audit/questions.ts holds 19 questions, held-out paraphrases, and one JSON answer schema each. - bench/audit/score.ts scores by exact match with no model judge: counts and paths by equality, times within 1 s, sets as sets, quotes by verbatim text plus a citation that resolves to the gold record. - bench/audit/cli.ts writes fixtures, writes gold separately, and scores any arm's answers JSON. The deterministic half runs in pnpm test; the model arms stay manual and are documented in bench/audit/README.md. Co-Authored-By: Claude Opus 5 (1M context) --- bench/audit/README.md | 173 +++++++ bench/audit/citations.ts | 166 +++++++ bench/audit/cli.ts | 147 ++++++ bench/audit/fixtures.test.ts | 362 +++++++++++++++ bench/audit/fixtures.ts | 861 +++++++++++++++++++++++++++++++++++ bench/audit/questions.ts | 308 +++++++++++++ bench/audit/score.test.ts | 241 ++++++++++ bench/audit/score.ts | 352 ++++++++++++++ package.json | 1 + tsconfig.json | 2 +- 10 files changed, 2612 insertions(+), 1 deletion(-) create mode 100644 bench/audit/README.md create mode 100644 bench/audit/citations.ts create mode 100644 bench/audit/cli.ts create mode 100644 bench/audit/fixtures.test.ts create mode 100644 bench/audit/fixtures.ts create mode 100644 bench/audit/questions.ts create mode 100644 bench/audit/score.test.ts create mode 100644 bench/audit/score.ts diff --git a/bench/audit/README.md b/bench/audit/README.md new file mode 100644 index 0000000..2ebc9a6 --- /dev/null +++ b/bench/audit/README.md @@ -0,0 +1,173 @@ +# Audit benchmark + +A public, dependency-free benchmark for one question: on a coding-agent session, does an +audit arm answer factual questions correctly and cheaply? + +The point of comparison is a fleet of general subagents turned loose on the raw session +files. That fleet is a real arm here, not a straw man, and it wins on some questions today. +Every claim that `traces` beats it has to come from a scored run of both arms on this +benchmark, with the costs reported. + +## What is in here + +| File | Owns | +|---|---| +| `fixtures.ts` | The generator. Writes three synthetic sessions and the gold answers. | +| `questions.ts` | The questions, their held-out paraphrases, and one JSON answer schema each. | +| `score.ts` | The exact-match scorer and the per-arm tally. | +| `citations.ts` | Resolves an answer's citation to the source record it names. | +| `cli.ts` | The runner: write fixtures, write gold, score an answers file. | + +Everything the fixtures contain is invented. No content from any real session appears in +this directory, in its tests, or in anything it generates. + +### Why the gold does not come from an adapter + +`fixtures.ts` records each planted fact at the moment it writes the record that carries it. +The answer key therefore comes from the plan, never from parsing the files back through +`src/adapters/`. An adapter defect shows up here as a wrong answer, which is the point; if +the gold went through the adapter, the same defect would quietly rewrite the answer key and +the benchmark would score nothing. + +`fixtures.test.ts` closes the other half of that loop: it re-derives every planted fact by +reading the generated JSONL directly, with no adapter and no access to the generator's +bookkeeping, and fails if the plan and the bytes disagree. + +## Sessions and what they plant + +**`codex-operator`** — a Codex operator session, over 600 spans, eight turns. + +- One command repeated past any single page of matches: 523 `labctl status` calls, two of + which name a run that does not exist. +- 16 `spawn_agent` calls, one of which fails on an agent thread limit. +- Three pull requests opened three ways: one inside an `exec` script, one straight from + `exec_command`, and one whose URL never appears in the create command's own output and + shows up only in a later `write_stdin` poll of the backgrounded process. +- Three merges, the same three ways, plus one earlier merge the harness refused. +- Runs launched, relaunched with different arguments after a failure, and cancelled, with a + cancel of a run id that does not exist. +- Injected text that is not a human turn: an `AGENTS.md` block, `environment_context` + blocks, and `subagent_notification` blocks, including two after the last human turn. +- A short last human turn (`ya?`) following a substantive one. +- `apply_patch` both as its own tool call and nested inside an `exec` script. +- One tool output over 16 KiB whose answer is its last line. +- A repeated `token_count` record with an unchanged cumulative total, so summing deltas + double-counts. + +**`codex-child`** — the session forked from one of those spawns. It starts with the +parent's history rewritten to the fork timestamp, so any count that includes inherited +records is wrong. + +**`claude`** — a Claude Code session with three `Task` subagents, each with its own +sidechain transcript, and `Bash` results carrying `is_error`, in the main transcript and +inside the subagents. + +## Failure classes + +`probes` on each question names the failure classes from the improvement brief it +exercises. + +| Class | What it is | Questions | +|---|---|---| +| F1 | Ordered human turns, told apart from injected text | `op.last-human-turn`, `op.corrections` | +| F2 | Enumeration over a session longer than one read | `op.subagents`, `op.runs`, `op.role`, `op.status-polls`, `op.exit-codes`, `op.pull-requests`, `child.own-work`, `claude.tasks`, `claude.bash` | +| F3 | Timestamps of specific records | `op.pull-requests`, `op.time-bounds`, `child.own-work` | +| F4 | Facts linked across spans and across sessions | `op.pull-requests`, `child.lineage` | +| F5 | Files a session changed | `op.local-copy`, `op.changed-files` | +| F6 | Verbatim tool output | `claude.first-bash-error` | +| F9 | Output past a truncation boundary | `op.large-output` | +| F11 | Derived fields, including the correct empty answer | `op.exit-codes`, `op.tokens`, `child.spawned` | + +## Scoring + +Exact match. No model judges any answer. + +- Counts, numbers, names, paths, booleans: equality. +- Times: correct within 1 s. Neighboring records in the fixtures are at least 2 s apart, so + the tolerance can never accept an adjacent record's time. +- Sets: set equality, order ignored, no missing and no extra member. +- Quotes: the text verbatim (whitespace-insensitive) **and** a citation that resolves to + the gold record. A citation may be `:`, a span id, or a + `trace:///span/` URI; span ids resolve through the source-record offsets the + adapters attach, so a span id counts only when the span really came from that record. +- A question is `correct` when every leaf is correct, `wrong` when none is, `partial` + otherwise. + +Two things are counted rather than averaged away. A leaf answered `null` where the gold has +a value is reported separately as a false "not in trace"; a confident wrong answer and a +refusal are different failures. And an unreported cost stays `missing` — it never becomes +zero. + +## Running it + +```sh +# The tree an arm may read. It does not contain the gold. +pnpm bench:audit fixtures --out /tmp/audit-bench + +# The answer key, written somewhere the arm cannot see. +pnpm bench:audit gold --out /tmp/audit-gold.json + +# Score an arm. +pnpm bench:audit score /tmp/arm-answers.json --fixtures /tmp/audit-bench --out /tmp/report.md +``` + +`--fixtures` is optional; without it the runner regenerates the tree in a temporary +directory. When it is given, every file in it must still match the generated bytes, so an +arm cannot be scored against a tree it edited. + +`prompts.jsonl` in the fixtures directory holds one row per question wording: the canonical +question at `variant: 0` and each held-out paraphrase above it, each with the exact prompt +text and the JSON Schema the answer must match. An arm that only handles the canonical +wordings is scored on those alone; the paraphrase tally is reported separately. + +### The answers file + +```json +{ + "arm": "subagent-fleet", + "notes": "5 general subagents, one per session, no traces CLI", + "answers": [ + { + "question": "op.status-polls", + "variant": 0, + "answer": { "status_commands": 523 }, + "wall_ms": 41200, + "model_calls": 18, + "tool_calls": 96, + "cost_usd": 0.83, + "cost_basis": "observed" + } + ] +} +``` + +`answer` is the object the question's schema describes, or `null` when the arm produced +none. `wall_ms`, `model_calls`, `tool_calls` and `cost_usd` are optional; leave one out +rather than guessing, because an omitted measure is reported as missing and a guessed one +is reported as a number. `cost_basis` says whether `cost_usd` was observed or estimated. + +## What `pnpm test` covers, and what it does not + +`pnpm test` runs the deterministic half: + +- `fixtures.test.ts` — the generator is byte-identical across runs, every planted fact is + present, and the gold equals a re-derivation of the fixtures that never touches the + generator's bookkeeping or an adapter. +- `score.test.ts` — the gold, submitted as an arm, scores every question correct through + the same path a real arm takes; and the scorer's leaf rules, citation resolution, + answers-file validation, tallies, and the runner end to end. + +The model arms are **manual**, on purpose: they cost money, they are not deterministic, and +CI must not depend on a model provider. Run them by hand and keep the answers files. + +Two arms are worth running against each other: + +1. **Subagent fleet.** Point a coding agent at the fixtures directory with no `traces` CLI + and let it spawn whatever subagents it wants. This is the baseline to beat. +2. **traces.** The same coding agent, allowed to use the `traces` CLI over the same + fixtures directory. + +Give both arms the same prompts from `prompts.jsonl`, the same model, and the same fixture +tree; record wall time, model calls, tool calls, and cost for each; then score both and +report the two tallies together. A single paired run supports a claim about that run, not a +general one — repeat before claiming an arm is better. diff --git a/bench/audit/citations.ts b/bench/audit/citations.ts new file mode 100644 index 0000000..152b87f --- /dev/null +++ b/bench/audit/citations.ts @@ -0,0 +1,166 @@ +/** + * Resolve an arm's citation to the source records it names. + * + * Gold quotes name a record by `:`, which any arm can produce from + * the raw files. A traces arm cites span ids instead; those resolve through the + * source-record offsets the traces adapters attach to each span, so a span id + * counts only when the span really came from the gold record. The gold itself + * never passes through an adapter. + */ + +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { ClaudeAdapter } from '../../src/adapters/claude.js' +import { CodexAdapter } from '../../src/adapters/codex.js' +import { SOURCE_ATTRIBUTE_PREFIX, sourceFileId, type SourceRecordReference } from '../../src/source-location.js' +import type { HarnessTraceAdapter } from '../../src/types.js' +import type { BenchManifest } from './fixtures.js' + +export interface RecordRef { + file: string + line: number +} + +export interface CitationIndex { + /** The records a citation names, or undefined when it names nothing known. */ + resolve(cite: string): readonly RecordRef[] | undefined + /** Whether a record holds the text in one of its decoded string values. */ + contains(ref: RecordRef, text: string): boolean +} + +/** Whitespace-insensitive form used for every text comparison. */ +export function normalizeText(text: string): string { + return text.replace(/\s+/g, ' ').trim() +} + +function stringValues(value: unknown, out: string[]): string[] { + if (typeof value === 'string') out.push(value) + else if (Array.isArray(value)) for (const item of value) stringValues(item, out) + else if (value && typeof value === 'object') for (const item of Object.values(value)) stringValues(item, out) + return out +} + +const TRACE_URI = /^trace:\/\/([^/]+)\/span\/([^/]+)$/ +const LINE_CITE = /^(.+):(\d+)$/ + +/** + * An index over the fixture files alone: it resolves `:` citations. + * `spans` adds span-id aliases, each mapped to the records its span came from. + */ +export function createCitationIndex( + files: ReadonlyMap, + spans: ReadonlyMap = new Map(), +): CitationIndex { + const lines = new Map() + const byBasename = new Map() + for (const [path, content] of files) { + lines.set(path, content.split('\n')) + const base = path.split('/').at(-1)! + byBasename.set(base, [...(byBasename.get(base) ?? []), path]) + } + const decoded = new Map() + + const fileFor = (raw: string): string | undefined => { + const path = raw.replace(/^\.\//, '').replace(/^fixtures\//, '') + if (files.has(path)) return path + const matches = byBasename.get(path.split('/').at(-1) ?? '') ?? [] + return matches.length === 1 ? matches[0] : undefined + } + + return { + resolve(cite) { + const trimmed = cite.trim() + const uri = TRACE_URI.exec(trimmed) + const spanKey = uri ? decodeURIComponent(uri[2]!) : trimmed + const bySpan = spans.get(spanKey) + if (bySpan) return bySpan + const lineCite = LINE_CITE.exec(trimmed) + if (!lineCite) return undefined + const file = fileFor(lineCite[1]!) + const line = Number(lineCite[2]) + if (!file || line < 1 || line > (lines.get(file)?.length ?? 0)) return undefined + return [{ file, line }] + }, + contains(ref, text) { + const key = `${ref.file}:${ref.line}` + let values = decoded.get(key) + if (!values) { + const raw = lines.get(ref.file)?.[ref.line - 1] ?? '' + let parsed: unknown = raw + try { + parsed = JSON.parse(raw) as unknown + } catch { + // A non-JSON line is compared as raw text. + } + values = stringValues(parsed, []).map(normalizeText) + decoded.set(key, values) + } + const wanted = normalizeText(text) + return wanted.length > 0 && values.some((value) => value.includes(wanted)) + }, + } +} + +function lineStarts(content: Buffer): number[] { + const starts = [0] + for (let index = 0; index < content.length; index += 1) if (content[index] === 0x0a) starts.push(index + 1) + return starts +} + +const ADAPTERS: Record = { + codex: new CodexAdapter(), + 'claude-code': new ClaudeAdapter(), +} + +/** + * Parse every fixture session through its traces adapter and map each span id + * (hex wire id and unambiguous readable source id) to the records it came from. + */ +export async function spanRecordMap(root: string, manifest: BenchManifest): Promise> { + const fileById = new Map() + for (const path of manifest.files) { + const bytes = await readFile(join(root, path)) + fileById.set(sourceFileId(join(root, path)), { path, starts: lineStarts(bytes) }) + } + const spans = new Map() + const readable = new Map() + for (const session of manifest.sessions) { + const adapter = ADAPTERS[session.harness] + if (!adapter) throw new Error(`no adapter for ${session.harness}`) + const parsed = await adapter.parse({ + harness: session.harness, + sessionId: session.sessionId, + path: join(root, session.path), + cwd: null, + mtimeMs: 0, + }, { captureSources: true }) + for (const span of parsed) { + const refs = new Map() + for (const [key, value] of Object.entries(span.attributes)) { + if (!key.startsWith(SOURCE_ATTRIBUTE_PREFIX) || typeof value !== 'string') continue + for (const ref of JSON.parse(value) as SourceRecordReference[]) { + const file = fileById.get(ref.sourceId) + if (!file) continue + const index = file.starts.indexOf(ref.recordOffset) + if (index >= 0) refs.set(`${file.path}:${index + 1}`, { file: file.path, line: index + 1 }) + } + } + const records = [...refs.values()] + spans.set(span.span_id, records) + for (const key of ['traces.codex.source_span_id', 'traces.claude.source_span_id']) { + const alias = span.attributes[key] + if (typeof alias === 'string') readable.set(alias, [...(readable.get(alias) ?? []), records]) + } + } + } + // A readable id shared by two sessions (a fork repeats its parent's call ids) names nothing. + for (const [alias, candidates] of readable) if (candidates.length === 1 && !spans.has(alias)) spans.set(alias, candidates[0]!) + return spans +} + +/** The full index an arm's answers are scored against. */ +export async function loadCitationIndex(root: string, manifest: BenchManifest): Promise { + const files = new Map() + for (const path of manifest.files) files.set(path, await readFile(join(root, path), 'utf8')) + return createCitationIndex(files, await spanRecordMap(root, manifest)) +} diff --git a/bench/audit/cli.ts b/bench/audit/cli.ts new file mode 100644 index 0000000..0983e67 --- /dev/null +++ b/bench/audit/cli.ts @@ -0,0 +1,147 @@ +#!/usr/bin/env node +/** + * The benchmark runner. + * + * tsx bench/audit/cli.ts fixtures --out + * tsx bench/audit/cli.ts gold --out + * tsx bench/audit/cli.ts score [--fixtures ] [--out ] [--json ] + * + * `fixtures` writes the tree an arm is allowed to read: the synthetic sessions, + * a manifest, and one prompt per question wording. The gold answers are written + * only by `gold`, into a separate path, so an arm's working directory can hold + * the sessions without holding the answer key. + * + * `score` is the whole grader. It compares an arm's answers JSON against the + * gold by exact match, with no model in the loop, and it rebuilds the fixture + * bytes itself: a `--fixtures` directory is accepted only when every file in it + * matches the generated bytes, so an arm cannot be scored against a tree it + * edited. + */ + +import { mkdtemp, mkdir, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { loadCitationIndex } from './citations.js' +import { generateBench, writeFixtures, type GeneratedBench } from './fixtures.js' +import { promptRows } from './questions.js' +import { parseAnswersFile, renderArmScore, scoreArm } from './score.js' + +const USAGE = [ + 'Usage:', + ' bench/audit/cli.ts fixtures --out ', + ' bench/audit/cli.ts gold --out ', + ' bench/audit/cli.ts score [--fixtures ] [--out ] [--json ]', +].join('\n') + +interface Args { + positional: string[] + flags: Map +} + +function parseArgs(argv: readonly string[]): Args { + const positional: string[] = [] + const flags = new Map() + for (let index = 0; index < argv.length; index += 1) { + const item = argv[index]! + if (!item.startsWith('--')) { + positional.push(item) + continue + } + const equals = item.indexOf('=') + if (equals > 0) { + flags.set(item.slice(2, equals), item.slice(equals + 1)) + continue + } + const value = argv[index + 1] + if (value === undefined || value.startsWith('--')) throw new Error(`${item} needs a value`) + flags.set(item.slice(2), value) + index += 1 + } + return { positional, flags } +} + +async function writeOut(path: string, content: string): Promise { + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, content) +} + +/** Write the fixture tree plus the manifest and prompts an arm needs. */ +async function writeArmInputs(root: string, bench: GeneratedBench): Promise { + await writeFixtures(root, bench) + await writeOut(join(root, 'manifest.json'), `${JSON.stringify(bench.manifest, null, 2)}\n`) + const prompts = promptRows(bench.manifest).map((row) => JSON.stringify(row)).join('\n') + await writeOut(join(root, 'prompts.jsonl'), `${prompts}\n`) +} + +/** + * A fixture directory is usable only when it still holds the generated bytes. + * Scoring an arm against a tree the arm could have rewritten would grade the + * tree, not the arm. + */ +async function assertUnmodified(root: string, bench: GeneratedBench): Promise { + for (const file of bench.files) { + const path = join(root, file.path) + const actual = await readFile(path, 'utf8').catch(() => undefined) + if (actual === undefined) throw new Error(`fixture directory is missing ${file.path}`) + if (actual !== file.content) throw new Error(`fixture file differs from the generated bytes: ${file.path}`) + } +} + +async function main(argv: readonly string[]): Promise { + const [command, ...rest] = argv + if (!command || command === '--help' || command === '-h') { + console.log(USAGE) + return command ? 0 : 1 + } + const { positional, flags } = parseArgs(rest) + const bench = generateBench() + + if (command === 'fixtures') { + const out = flags.get('out') + if (!out) throw new Error('fixtures needs --out ') + await writeArmInputs(out, bench) + console.log(`wrote ${bench.files.length} session files, manifest.json and prompts.jsonl to ${out}`) + return 0 + } + + if (command === 'gold') { + const out = flags.get('out') + if (!out) throw new Error('gold needs --out ') + await writeOut(out, `${JSON.stringify(bench.gold, null, 2)}\n`) + console.log(`wrote gold answers for ${Object.keys(bench.gold).length} questions to ${out}`) + return 0 + } + + if (command === 'score') { + const answersPath = positional[0] + if (!answersPath) throw new Error('score needs an answers JSON path') + const file = parseAnswersFile(JSON.parse(await readFile(answersPath, 'utf8')) as unknown) + let root = flags.get('fixtures') + if (root) await assertUnmodified(root, bench) + else { + root = await mkdtemp(join(tmpdir(), 'traces-bench-audit-')) + await writeFixtures(root, bench) + } + const index = await loadCitationIndex(root, bench.manifest) + const score = scoreArm(file, bench.gold, index) + const report = renderArmScore(score) + const out = flags.get('out') + if (out) await writeOut(out, report) + else console.log(report) + const json = flags.get('json') + if (json) await writeOut(json, `${JSON.stringify(score, null, 2)}\n`) + return 0 + } + + throw new Error(`unknown command ${command}\n${USAGE}`) +} + +main(process.argv.slice(2)).then( + (code) => { + process.exitCode = code + }, + (error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 + }, +) diff --git a/bench/audit/fixtures.test.ts b/bench/audit/fixtures.test.ts new file mode 100644 index 0000000..6bb16bd --- /dev/null +++ b/bench/audit/fixtures.test.ts @@ -0,0 +1,362 @@ +/** + * The generator is checked two ways. + * + * Determinism: the same seed must give the same bytes, or an arm scored today + * and an arm scored tomorrow answered different questions. + * + * Gold equality: every planted fact is recomputed here by reading the generated + * JSONL back, with no access to the generator's bookkeeping and no traces + * adapter in the path. If the plan and the bytes ever disagree, the answer key + * is wrong, and that has to fail here rather than silently mark a correct arm + * wrong. + */ + +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, describe, expect, it } from 'vitest' +import { ClaudeAdapter } from '../../src/adapters/claude.js' +import { CodexAdapter } from '../../src/adapters/codex.js' +import { generateBench, writeFixtures } from './fixtures.js' +import { QUESTIONS, answerJsonSchema, promptRows } from './questions.js' + +const bench = generateBench() +const fileByPath = new Map(bench.files.map((file) => [file.path, file.content])) + +const operatorPath = bench.manifest.sessions.find((session) => session.id === 'codex-operator')!.path +const childPath = bench.manifest.sessions.find((session) => session.id === 'codex-child')!.path +const claudePath = bench.manifest.sessions.find((session) => session.id === 'claude')!.path + +interface CodexRow { + line: number + timestamp: string + type: string + payload: Record +} + +function codexRows(path: string): CodexRow[] { + return fileByPath.get(path)!.split('\n').filter((line) => line.length > 0).map((line, index) => { + const row = JSON.parse(line) as { timestamp: string; type: string; payload: Record } + return { line: index + 1, ...row } + }) +} + +const operator = codexRows(operatorPath) + +/** Tool calls paired with the output record that answered them. */ +interface CodexCall { + name: string + argument: string + call: CodexRow + output: string +} + +function codexCalls(rows: readonly CodexRow[]): CodexCall[] { + const pending = new Map() + const calls: CodexCall[] = [] + for (const row of rows) { + const payload = row.payload + const type = payload.type + if (type === 'function_call' || type === 'custom_tool_call') { + pending.set(String(payload.call_id), { + name: String(payload.name), + argument: String(type === 'function_call' ? payload.arguments : payload.input), + call: row, + }) + } + if (type === 'function_call_output' || type === 'custom_tool_call_output') { + const started = pending.get(String(payload.call_id)) + if (started) calls.push({ ...started, output: String(payload.output ?? '') }) + } + } + return calls +} + +const operatorCalls = codexCalls(operator) +const commandOf = (call: CodexCall): string => String((JSON.parse(call.argument) as { cmd?: string }).cmd ?? '') +const execCalls = operatorCalls.filter((call) => call.name === 'exec_command') +const scripts = operatorCalls.filter((call) => call.name === 'exec') + +/** Text of a user message, or undefined for any other record. */ +function userText(row: CodexRow): string | undefined { + if (row.payload.type !== 'message' || row.payload.role !== 'user') return undefined + const content = row.payload.content as Array<{ text?: string }> + return content.map((part) => part.text ?? '').join('') +} + +/** A human turn is a user message the harness did not inject. */ +const humanTurns = operator + .map((row) => ({ row, text: userText(row) })) + .filter((item): item is { row: CodexRow; text: string } => item.text !== undefined) + .filter((item) => !item.text.startsWith('<') && !item.text.startsWith('#')) + +describe('audit benchmark generator', () => { + it('produces the same bytes on every run', () => { + const again = generateBench() + expect(again.files).toEqual(bench.files) + expect(again.manifest).toEqual(bench.manifest) + expect(again.gold).toEqual(bench.gold) + }) + + it('writes exactly the manifest files, byte for byte', async () => { + const root = await mkdtemp(join(tmpdir(), 'traces-bench-write-')) + try { + await writeFixtures(root, bench) + for (const path of bench.manifest.files) { + expect(await readFile(join(root, path), 'utf8')).toBe(fileByPath.get(path)) + } + expect(bench.manifest.files).toEqual([...bench.manifest.files].sort()) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('asks exactly the questions the gold answers', () => { + expect(QUESTIONS.map((question) => question.id).sort()).toEqual(Object.keys(bench.gold).sort()) + expect(new Set(QUESTIONS.map((question) => question.id)).size).toBe(QUESTIONS.length) + for (const question of QUESTIONS) { + const schema = answerJsonSchema(question) as { required: string[] } + expect(schema.required.sort()).toEqual(Object.keys(bench.gold[question.id]!).sort()) + } + }) + + it('gives every question wording a prompt naming its session', () => { + const rows = promptRows(bench.manifest) + expect(rows).toHaveLength(QUESTIONS.reduce((sum, question) => sum + 1 + question.paraphrases.length, 0)) + for (const row of rows) { + expect(row.prompt).toContain(row.sessionPath) + expect(row.heldOut).toBe(row.variant > 0) + } + }) +}) + +describe('planted facts in the operator session', () => { + it('is long enough that no single read covers it', async () => { + const root = await mkdtemp(join(tmpdir(), 'traces-bench-spans-')) + try { + await writeFixtures(root, bench) + const spans = await new CodexAdapter().parse({ + harness: 'codex', + sessionId: bench.manifest.sessions[0]!.sessionId, + path: join(root, operatorPath), + cwd: null, + mtimeMs: 0, + }) + expect(spans.length).toBeGreaterThan(600) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('repeats one command past any single page of matches', () => { + const polls = execCalls.filter((call) => commandOf(call).startsWith('labctl status ')) + expect(polls.length).toBeGreaterThan(500) + expect(bench.gold['op.status-polls']).toEqual({ status_commands: polls.length }) + }) + + it('spawns sixteen agents, one of which fails', () => { + const spawns = operatorCalls.filter((call) => call.name === 'spawn_agent') + expect(spawns).toHaveLength(16) + expect(spawns.filter((call) => call.output.startsWith('spawn_agent failed'))).toHaveLength(1) + const succeeded = spawns.filter((call) => !call.output.startsWith('spawn_agent failed')) + const names = [...new Set(succeeded.map((call) => String((JSON.parse(call.argument) as { task_name: string }).task_name)))].sort() + expect(bench.gold['op.subagents']).toEqual({ spawn_calls: 16, failed_spawns: 1, task_names: names }) + }) + + it('opens three pull requests three different ways', () => { + const numbersIn = (text: string): number[] => [...text.matchAll(/\/pull\/(\d+)/g)].map((match) => Number(match[1])) + const stdinCalls = operatorCalls.filter((call) => call.name === 'write_stdin') + // One inside an exec script, one straight from exec_command, one that only a later poll reveals. + expect(scripts.flatMap((call) => numbersIn(call.output))).toEqual([41]) + expect(execCalls.filter((call) => commandOf(call).startsWith('gh pr create')).flatMap((call) => numbersIn(call.output))).toEqual([42]) + expect(stdinCalls.flatMap((call) => numbersIn(call.output))).toEqual([43]) + const create43 = execCalls.find((call) => commandOf(call).includes('docs(retry): budget guide')) + expect(create43).toBeDefined() + expect(numbersIn(create43!.output)).toEqual([]) + expect((bench.gold['op.pull-requests']!.prs as Array<{ number: number }>).map((pr) => pr.number)).toEqual([41, 42, 43]) + }) + + it('merges three pull requests three different ways', () => { + const merged = (text: string): number[] => [...text.matchAll(/Squashed and merged pull request [^#]+#(\d+)/g)].map((match) => Number(match[1])) + expect(execCalls.flatMap((call) => merged(call.output))).toEqual([41]) + expect(operatorCalls.filter((call) => call.name === 'write_stdin').flatMap((call) => merged(call.output))).toEqual([42]) + expect(scripts.flatMap((call) => merged(call.output))).toEqual([43]) + const prs = bench.gold['op.pull-requests']!.prs as Array<{ number: number; merged_at?: string }> + expect(prs.filter((pr) => pr.merged_at)).toHaveLength(3) + // A merge that the harness refused must not count as the merge. + expect(execCalls.some((call) => commandOf(call) === 'gh pr merge 41 --squash' && call.output.includes('not mergeable'))).toBe(true) + }) + + it('repeats and cancels run commands', () => { + const launches = execCalls.filter((call) => commandOf(call).startsWith('labctl run ')) + const specs = launches.map((call) => commandOf(call).split(' ')[2]!) + expect(new Set(specs).size).toBeLessThan(specs.length) + const cancels = execCalls.filter((call) => commandOf(call).startsWith('labctl cancel ')) + const cancelled = cancels.filter((call) => call.output.includes('Process exited with code 0')) + expect(bench.gold['op.runs']!.cancelled).toBe(cancelled.length) + expect(cancels.length).toBeGreaterThan(cancelled.length) + }) + + it('ends with a short human turn after a substantive one, then injected text', () => { + const last = humanTurns.at(-1)! + const previous = humanTurns.at(-2)! + expect(last.text.length).toBeLessThan(8) + expect(previous.text.length).toBeGreaterThan(40) + const injectedAfter = operator.filter((row) => row.line > last.row.line && userText(row)?.startsWith('<')) + expect(injectedAfter.length).toBeGreaterThan(0) + expect(bench.gold['op.last-human-turn']).toEqual({ + last: { text: last.text, cite: `${operatorPath}:${last.row.line}` }, + last_at: last.row.timestamp, + previous: { text: previous.text, cite: `${operatorPath}:${previous.row.line}` }, + }) + }) + + it('runs apply_patch inside an exec script as well as on its own', () => { + expect(scripts.some((call) => call.argument.includes('tools.apply_patch('))).toBe(true) + const paths = new Set() + for (const call of operatorCalls) { + const text = call.name === 'apply_patch' ? call.argument : call.name === 'exec' ? call.argument : '' + for (const match of text.matchAll(/^\*\*\* (?:Add|Update|Delete) File: (.+)$/gm)) paths.add(match[1]!) + } + expect(bench.gold['op.changed-files']).toEqual({ paths: [...paths].sort() }) + }) + + it('holds one tool output past sixteen kibibytes', () => { + const large = operatorCalls.filter((call) => Buffer.byteLength(call.output) > 16 * 1024) + expect(large).toHaveLength(1) + const lastLine = large[0]!.output.trimEnd().split('\n').at(-1) + expect((bench.gold['op.large-output']!.last_line as { text: string }).text).toBe(lastLine) + }) + + it('records the counts the gold reports for exits, tokens and time bounds', () => { + const exits = execCalls + .map((call) => Number(/Process exited with code (-?\d+)/.exec(call.output)?.[1] ?? 0)) + .filter((code) => code !== 0) + expect(bench.gold['op.exit-codes']).toEqual({ + nonzero_exec_commands: exits.length, + codes: [...new Set(exits)].sort((a, b) => a - b), + }) + const totals = operator + .filter((row) => row.payload.type === 'token_count') + .map((row) => (row.payload.info as { total_token_usage: Record }).total_token_usage) + .at(-1)! + expect(bench.gold['op.tokens']).toEqual({ + input_tokens: totals.input_tokens, + cached_input_tokens: totals.cached_input_tokens, + output_tokens: totals.output_tokens, + }) + expect(bench.gold['op.time-bounds']).toEqual({ + first_record_at: operator[0]!.timestamp, + last_record_at: operator.at(-1)!.timestamp, + }) + }) + + it('quotes every correction at the record it was typed in', () => { + const corrections = bench.gold['op.corrections']!.corrections as Array<{ text: string; cite: string }> + expect(corrections.length).toBeGreaterThan(1) + for (const correction of corrections) { + const line = Number(correction.cite.split(':').at(-1)) + expect(userText(operator[line - 1]!)).toBe(correction.text) + } + }) +}) + +describe('planted facts in the forked child session', () => { + const child = codexRows(childPath) + const meta = child[0]!.payload as Record + + it('names its parent and repeats the parent history it forked from', () => { + expect(meta.parent_thread_id).toBe(bench.manifest.sessions[0]!.sessionId) + expect(bench.gold['child.lineage']).toEqual({ parent_session_id: meta.parent_thread_id, agent_path: meta.agent_path }) + const inherited = child.slice(1).filter((row) => row.timestamp === child[0]!.timestamp) + expect(inherited.length).toBeGreaterThan(10) + const parentBodies = new Set(operator.map((row) => JSON.stringify(row.payload))) + for (const row of inherited) expect(parentBodies.has(JSON.stringify(row.payload))).toBe(true) + }) + + it('counts only the work it did after the fork', () => { + const own = child.filter((row) => row.timestamp !== child[0]!.timestamp) + const calls = own.filter((row) => row.payload.type === 'function_call' || row.payload.type === 'custom_tool_call') + const failed = codexCalls(own).filter((call) => /Process exited with code (?!0)/.test(call.output)) + expect(bench.gold['child.own-work']).toEqual({ + task_started_at: own.find((row) => row.payload.type === 'task_started')!.timestamp, + own_tool_calls: calls.length, + failed_commands: failed.length, + }) + expect(bench.gold['child.spawned']).toEqual({ spawned_session_ids: [] }) + }) +}) + +describe('planted facts in the Claude session', () => { + interface ClaudeRow { + line: number + type: string + isSidechain: boolean + message: { content: unknown } + } + const read = (path: string): ClaudeRow[] => + fileByPath.get(path)!.split('\n').filter((line) => line.length > 0) + .map((line, index) => ({ line: index + 1, ...(JSON.parse(line) as Omit) })) + const main = read(claudePath) + const subagentFiles = bench.manifest.files.filter((path) => path.includes('/subagents/') && path.endsWith('.jsonl')) + const blocks = (rows: readonly ClaudeRow[]): Array> => + rows.flatMap((row) => (Array.isArray(row.message.content) ? row.message.content as Array> : [])) + + it('launches Task subagents that each get their own transcript', () => { + const tasks = blocks(main).filter((block) => block.type === 'tool_use' && block.name === 'Task') + expect(tasks).toHaveLength(3) + expect(subagentFiles).toHaveLength(3) + expect(bench.gold['claude.tasks']).toEqual({ + task_calls: tasks.length, + subagent_types: [...new Set(tasks.map((task) => (task.input as { subagent_type: string }).subagent_type))].sort(), + descriptions: tasks.map((task) => (task.input as { description: string }).description).sort(), + }) + for (const row of subagentFiles.flatMap(read)) expect(row.isSidechain).toBe(true) + }) + + it('counts Bash errors in the main transcript and inside the subagents', () => { + const bashIds = (rows: readonly ClaudeRow[]): Set => + new Set(blocks(rows).filter((block) => block.type === 'tool_use' && block.name === 'Bash').map((block) => String(block.id))) + const errors = (rows: readonly ClaudeRow[], ids: ReadonlySet): number => + blocks(rows).filter((block) => block.type === 'tool_result' && block.is_error === true && ids.has(String(block.tool_use_id))).length + const subagentRows = subagentFiles.flatMap(read) + const mainIds = bashIds(main) + const subIds = bashIds(subagentRows) + const subErrors = errors(subagentRows, subIds) + expect(bench.gold['claude.bash']).toEqual({ + bash_calls: mainIds.size + subIds.size, + failed_bash_calls: errors(main, mainIds) + subErrors, + failed_in_subagents: subErrors, + }) + expect(subErrors).toBeGreaterThan(0) + }) + + it('quotes the first failing Bash output from the record that carries it', () => { + const quote = bench.gold['claude.first-bash-error']!.output as { text: string; cite: string } + const line = Number(quote.cite.split(':').at(-1)) + const results = blocks([main[line - 1]!]).filter((block) => block.is_error === true) + expect(results).toHaveLength(1) + expect(results[0]!.content).toBe(quote.text) + }) +}) + +describe('the Claude adapter reads the generated session', () => { + let root: string | undefined + afterAll(async () => { + if (root) await rm(root, { recursive: true, force: true }) + }) + + it('parses the main transcript into spans', async () => { + root = await mkdtemp(join(tmpdir(), 'traces-bench-claude-')) + await writeFixtures(root, bench) + const session = bench.manifest.sessions.find((item) => item.harness === 'claude-code')! + const spans = await new ClaudeAdapter().parse({ + harness: 'claude-code', + sessionId: session.sessionId, + path: join(root, session.path), + cwd: null, + mtimeMs: 0, + }) + expect(spans.length).toBeGreaterThan(0) + }) +}) diff --git a/bench/audit/fixtures.ts b/bench/audit/fixtures.ts new file mode 100644 index 0000000..922ee89 --- /dev/null +++ b/bench/audit/fixtures.ts @@ -0,0 +1,861 @@ +/** + * Deterministic generator for the public audit benchmark. + * + * Every session here is synthetic. Each planted fact is recorded while its + * record is written, so the gold answers come from the plan, never from parsing + * the files back. The gold therefore does not depend on any traces adapter, and + * an adapter defect shows up as a wrong answer instead of a wrong answer key. + * + * Neighboring records are at least 2 s apart, so the scorer's 1 s tolerance for + * times can never accept the time of an adjacent record. + */ + +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' + +export type BenchSessionId = 'codex-operator' | 'codex-child' | 'claude' + +export interface BenchSession { + id: BenchSessionId + harness: 'codex' | 'claude-code' + /** Path relative to the fixture root. */ + path: string + sessionId: string +} + +export interface BenchManifest { + version: 1 + sessions: BenchSession[] + /** Every file under the fixture root, relative, sorted. */ + files: string[] +} + +/** Gold answers keyed by question id, in the answer shape an arm submits. */ +export type GoldAnswers = Record> + +export interface BenchFile { + path: string + content: string +} + +export interface GeneratedBench { + manifest: BenchManifest + files: BenchFile[] + gold: GoldAnswers +} + +const SEED = 0x5eed_a0d1 +const OPERATOR_START_MS = Date.UTC(2026, 2, 14, 9, 0, 0) +const CLAUDE_START_MS = Date.UTC(2026, 2, 15, 14, 0, 0) +const CODEX_MODEL = 'gpt-5-codex' +const CLAUDE_MODEL = 'claude-sonnet-4-5' +const REPO = 'acme/orbit' +const OPERATOR_CWD = '/work/orbit' +const CLAUDE_CWD = '/work/ledger' +/** More than 500, so a paged search over this command reports more hits than one page holds. */ +const STATUS_POLLS = 523 + +function mulberry32(seed: number): () => number { + let state = seed >>> 0 + return () => { + state = (state + 0x6d2b79f5) >>> 0 + let t = state + t = Math.imul(t ^ (t >>> 15), t | 1) + t ^= t + Math.imul(t ^ (t >>> 7), t | 61) + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296 + } +} + +type Random = () => number + +function hex(random: Random, length: number): string { + let out = '' + for (let index = 0; index < length; index += 1) out += Math.floor(random() * 16).toString(16) + return out +} + +function base62(random: Random, length: number): string { + const alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' + let out = '' + for (let index = 0; index < length; index += 1) out += alphabet[Math.floor(random() * alphabet.length)] + return out +} + +/** A UUIDv7 whose timestamp field is `ms`, as Codex session and turn ids are. */ +function uuidV7(ms: number, random: Random): string { + const time = ms.toString(16).padStart(12, '0') + const variant = (8 + Math.floor(random() * 4)).toString(16) + return `${time.slice(0, 8)}-${time.slice(8, 12)}-7${hex(random, 3)}-${variant}${hex(random, 3)}-${hex(random, 12)}` +} + +function uuidV4(random: Random): string { + const variant = (8 + Math.floor(random() * 4)).toString(16) + return `${hex(random, 8)}-${hex(random, 4)}-4${hex(random, 3)}-${variant}${hex(random, 3)}-${hex(random, 12)}` +} + +class Clock { + constructor(private ms: number, private readonly random: Random) {} + + get now(): number { + return this.ms + } + + next(minMs = 2_000, maxMs = 12_000): string { + this.ms += minMs + Math.floor(this.random() * (maxMs - minMs)) + return new Date(this.ms).toISOString() + } + + /** Move past `ms` so the next record follows everything written on another clock. */ + passAt(ms: number): void { + this.ms = Math.max(this.ms, ms) + } +} + +/** Where a planted record landed: its 1-based line and its timestamp. */ +interface Placed { + line: number + at: string +} + +interface Call extends Placed { + callId: string + kind: 'function' | 'custom' +} + +class Jsonl { + readonly rows: string[] = [] + + push(row: unknown): number { + this.rows.push(JSON.stringify(row)) + return this.rows.length + } + + text(): string { + return `${this.rows.join('\n')}\n` + } +} + +const cite = (file: string, line: number): string => `${file}:${line}` +const quote = (file: string, placed: Placed, text: string) => ({ text, cite: cite(file, placed.line) }) + +class CodexWriter { + readonly out = new Jsonl() + + constructor(readonly clock: Clock, private readonly random: Random) {} + + row(type: string, payload: Record, at = this.clock.next()): Placed { + return { line: this.out.push({ timestamp: at, type, payload }), at } + } + + message(role: 'user' | 'assistant', text: string): Placed { + return this.row('response_item', { + type: 'message', + role, + content: [{ type: role === 'user' ? 'input_text' : 'output_text', text }], + }) + } + + call(name: string, args: Record): Call { + const callId = `call_${base62(this.random, 24)}` + const placed = this.row('response_item', { type: 'function_call', name, arguments: JSON.stringify(args), call_id: callId }) + return { ...placed, callId, kind: 'function' } + } + + custom(name: string, input: string): Call { + const callId = `call_${base62(this.random, 24)}` + const placed = this.row('response_item', { type: 'custom_tool_call', status: 'completed', call_id: callId, name, input }) + return { ...placed, callId, kind: 'custom' } + } + + output(call: Call, output: string): Placed { + return this.row('response_item', { + type: call.kind === 'function' ? 'function_call_output' : 'custom_tool_call_output', + call_id: call.callId, + output, + }) + } +} + +/** The shell-tool output envelope Codex writes for `exec_command` and `write_stdin`. */ +function shellOutput(random: Random, result: number | { running: number }, body: string): string { + const status = typeof result === 'number' + ? `Process exited with code ${result}` + : `Process running with session ID ${result.running}` + return [ + `Chunk ID: ${hex(random, 6)}`, + `Wall time: ${(0.2 + random() * 4).toFixed(4)} seconds`, + status, + `Original token count: ${Math.max(1, Math.ceil(body.length / 4))}`, + 'Output:', + body, + ].join('\n') +} + +const PATCH_ACTIONS: Record = { Add: 'A', Update: 'M', Delete: 'D' } + +/** The summary Codex prints after a successful patch. */ +function patchSummary(text: string): string { + const lines = [...text.matchAll(/^\*\*\* (Add|Update|Delete) File: (.+)$/gm)].map((match) => `${PATCH_ACTIONS[match[1]!]} ${match[2]}`) + return `Success. Updated the following files:\n${lines.join('\n')}\n` +} + +interface TokenUsage { + input_tokens: number + cached_input_tokens: number + output_tokens: number + reasoning_output_tokens: number + total_tokens: number +} + +interface Spawn { + name: string + failed: boolean +} + +interface PullRequest { + created_at: string + merged_at?: string + reviewed_before_merge?: boolean +} + +interface OperatorPlan { + file: string + sessionId: string + rows: string[] + first: Placed + last: Placed + child: { sessionId: string; agentPath: string; forkRows: string[]; spawnAtMs: number } + gold: GoldAnswers +} + +function operatorSession(random: Random): OperatorPlan { + const clock = new Clock(OPERATOR_START_MS, random) + const w = new CodexWriter(clock, random) + const sessionId = uuidV7(OPERATOR_START_MS, random) + const file = `codex/sessions/2026/03/14/rollout-2026-03-14T09-00-00-${sessionId}.jsonl` + const prUrl = (number: number): string => `https://github.com/${REPO}/pull/${number}` + + const humans: Array = [] + const corrections: Array = [] + const spawns: Spawn[] = [] + const prs = new Map() + const changedPaths = new Set() + const runs = { launched: 0, failed: 0, cancelled: 0, specs: new Set(), betaVariants: new Set() } + const exits: number[] = [] + const launchedIds: string[] = [] + let statusPolls = 0 + let toolCalls = 0 + let turnId = '' + let child: OperatorPlan['child'] | undefined + const cumulative: TokenUsage = { input_tokens: 0, cached_input_tokens: 0, output_tokens: 0, reasoning_output_tokens: 0, total_tokens: 0 } + let lastTokenEvent: Record | undefined + + const tokenCount = (): void => { + const input = 20_000 + Math.floor(random() * 40_000) + const output = 200 + Math.floor(random() * 1_800) + const last: TokenUsage = { + input_tokens: input, + cached_input_tokens: Math.floor(input * 0.85), + output_tokens: output, + reasoning_output_tokens: Math.floor(output * 0.3), + total_tokens: input + output, + } + for (const key of Object.keys(cumulative) as Array) cumulative[key] += last[key] + lastTokenEvent = { type: 'token_count', info: { last_token_usage: last, total_token_usage: { ...cumulative }, model_context_window: 272_000 } } + w.row('event_msg', lastTokenEvent) + } + const afterTool = (): void => { + toolCalls += 1 + if (toolCalls % 8 === 0) tokenCount() + } + const startTurn = (): void => { + const at = clock.next() + turnId = uuidV7(Date.parse(at), random) + w.row('event_msg', { type: 'task_started', turn_id: turnId, started_at: Math.floor(Date.parse(at) / 1_000), model_context_window: 272_000 }, at) + w.row('turn_context', { cwd: OPERATOR_CWD, approval_policy: 'on-request', sandbox_policy: { mode: 'workspace-write' }, model: CODEX_MODEL }) + } + const endTurn = (message: string): void => { + w.message('assistant', message) + w.row('event_msg', { type: 'task_complete', turn_id: turnId, last_agent_message: message }) + } + const human = (text: string, correction = false): void => { + const placed = { ...w.message('user', text), text } + humans.push(placed) + if (correction) corrections.push(placed) + } + const exec = (cmd: string, result: number | { running: number }, body: string): Call & { output: Placed } => { + const call = w.call('exec_command', { cmd, workdir: OPERATOR_CWD, yield_time_ms: 10_000 }) + const output = w.output(call, shellOutput(random, result, body)) + if (typeof result === 'number' && result !== 0) exits.push(result) + // Counted here, so the two `labctl status` calls that name a run that does not exist count too. + if (cmd.startsWith('labctl status ')) statusPolls += 1 + afterTool() + return { ...call, output } + } + const writeStdin = (sessionIdValue: number, result: number | { running: number }, body: string): Call => { + const call = w.call('write_stdin', { session_id: sessionIdValue, chars: '', yield_time_ms: 5_000 }) + w.output(call, shellOutput(random, result, body)) + afterTool() + return call + } + const script = (source: string, body: string): Call => { + const call = w.custom('exec', source) + w.output(call, `Script completed\nWall time ${(0.5 + random() * 30).toFixed(1)} seconds\nOutput:\n${body}`) + afterTool() + return call + } + const patch = (text: string, paths: readonly string[]): Call => { + const call = w.custom('apply_patch', text) + w.output(call, JSON.stringify({ output: patchSummary(text), metadata: { exit_code: 0, duration_seconds: 0.1 } })) + for (const path of paths) changedPaths.add(path) + afterTool() + return call + } + const spawn = (name: string, message: string, failure?: string): void => { + const agentPath = `/root/${name}` + const call = w.call('spawn_agent', { task_name: agentPath, message }) + let agentId: string | undefined + if (failure) { + w.output(call, `spawn_agent failed: ${failure}`) + } else { + agentId = uuidV7(Date.parse(call.at) + 500, random) + w.output(call, JSON.stringify({ agent_id: agentId, task_name: agentPath, nickname: `Agent-${name.slice(0, 4)}` })) + } + spawns.push({ name: agentPath, failed: Boolean(failure) }) + if (name === 'flaky_test_hunt' && agentId) { + // A forked child starts with the parent's history up to the spawn call. + child = { sessionId: agentId, agentPath, forkRows: w.out.rows.slice(1, call.line - 1), spawnAtMs: Date.parse(call.at) } + } + afterTool() + } + const launch = (spec: string, args: string, result: number, body: string, runId?: string): void => { + exec(`labctl run ${spec}${args}`, result, body) + if (result !== 0) { + runs.failed += 1 + return + } + runs.launched += 1 + runs.specs.add(spec) + if (runId) launchedIds.push(runId) + } + const poll = (count: number): void => { + for (let index = 0; index < count; index += 1) { + const runId = launchedIds[Math.floor(random() * launchedIds.length)] ?? 'a-01' + const step = 1 + Math.floor(random() * 40) + exec(`labctl status ${runId}`, 0, `${runId} running step=${step}/40\n`) + } + } + + const first = w.row('session_meta', { + id: sessionId, + timestamp: new Date(OPERATOR_START_MS).toISOString(), + cwd: OPERATOR_CWD, + originator: 'codex_cli_rs', + cli_version: '0.150.0', + source: 'cli', + model_provider: 'openai', + }, new Date(OPERATOR_START_MS).toISOString()) + w.message('user', '# AGENTS.md instructions for /work/orbit\n\n\nRun pnpm test before every commit.\nOpen one pull request per change.\n') + w.message('user', '\n /work/orbit\n on-request\n workspace-write\n zsh\n') + + // Turn 1: fix, open three pull requests, spawn the first helpers. + startTurn() + human('Ship the retry budget change. Open a PR for each part, merge each one once its checks pass, then start the alpha, beta and gamma sweeps.') + w.message('assistant', 'Checking the tree and the current test result first.') + exec('git status --short', 0, ' M packages/retry/src/budget.ts\n') + exec('pnpm test --filter @acme/retry', 1, 'FAIL packages/retry/test/budget.test.ts\n expected 3 attempts, received 4\n') + patch('*** Begin Patch\n*** Update File: packages/retry/src/budget.ts\n@@\n- return attempts <= budget.max\n+ return attempts < budget.max\n*** End Patch', ['packages/retry/src/budget.ts']) + exec('pnpm test --filter @acme/retry', 0, 'Test Files 6 passed (6)\n') + exec('rg -n "export async function runGraph" node_modules/@acme/runtime/dist', 0, 'node_modules/@acme/runtime/dist/graph.js:14:export async function runGraph(graph, options = {}) {\n') + spawn('schema_audit', 'Check every retry config schema for fields the budget change removed.') + spawn('flaky_test_hunt', 'Find which retry tests fail intermittently and why.') + spawn('cache_probe', 'Measure cache hit rate of the retry lookup table.', 'agent thread limit reached (2 running, limit 2)') + spawn('docs_sweep', 'List docs pages that mention the old retry attempts default.') + const pr41 = script([ + 'const push = await tools.exec_command({ cmd: "git push -u origin feat/retry-budget" })', + 'const pr = await tools.exec_command({ cmd: "gh pr create --title \\"feat(retry): cap attempts per budget\\" --body-file .github/pr-retry.md" })', + 'await tools.update_plan({ plan: [{ step: "Open the retry pull request", status: "completed" }] })', + 'text(push.output + pr.output)', + ].join('\n'), `branch 'feat/retry-budget' set up to track 'origin/feat/retry-budget'.\n${prUrl(41)}\n`) + prs.set(41, { created_at: pr41.at }) + const pr42 = exec('gh pr create --title "feat(retry-cli): expose the budget flag" --fill', 0, `${prUrl(42)}\n`) + prs.set(42, { created_at: pr42.at }) + exec('pnpm dev --filter orbit-web', { running: 51200 }, 'ready on http://localhost:5173\n') + // The create command's own output has no URL; only a later poll of its process does. + const pr43 = exec('gh pr create --title "docs(retry): budget guide" --fill', { running: 73012 }, `Creating pull request for docs/retry-guide into main in ${REPO}\n`) + prs.set(43, { created_at: pr43.at }) + writeStdin(51200, { running: 51200 }, 'hmr update /src/app.tsx\n') + exec('gh pr list --state open', 0, '40\tchore: bump dependencies\tdeps/bump\tOPEN\n41\tfeat(retry): cap attempts per budget\tfeat/retry-budget\tOPEN\n42\tfeat(retry-cli): expose the budget flag\tfeat/retry-cli\tOPEN\n') + exec('gh pr merge 41 --squash', 1, `X Pull request ${REPO}#41 is not mergeable: required status checks have not passed\n`) + endTurn('PR 41 is not mergeable yet; its checks are still running.') + + // Turn 2: merge 41 properly, link 43, launch runs. + startTurn() + human("no, don't merge before the checks finish. wait for them next time.", true) + w.message('assistant', 'Understood. Waiting for the checks on 41 before merging.') + exec('gh pr checks 41 --watch', 0, 'All checks were successful\n') + exec('gh pr view 41 --json reviews', 0, '{"reviews":[]}\n') + const merged41 = exec('gh pr merge 41 --squash --delete-branch', 0, `Squashed and merged pull request ${REPO}#41 (feat(retry): cap attempts per budget)\nDeleted branch feat/retry-budget\n`) + prs.set(41, { ...prs.get(41)!, merged_at: merged41.at, reviewed_before_merge: false }) + writeStdin(73012, 0, `${prUrl(43)}\n`) + launch('alpha-sweep', '', 0, 'started run a-01 (spec alpha-sweep)\n', 'a-01') + launch('alpha-sweep', '', 0, 'started run a-02 (spec alpha-sweep)\n', 'a-02') + launch('beta-probe', ' --variant v1', 0, 'started run b-01 (spec beta-probe, variant v1)\n', 'b-01') + runs.betaVariants.add('v1') + launch('beta-probe', ' --variant v2', 0, 'started run b-02 (spec beta-probe, variant v2)\n', 'b-02') + runs.betaVariants.add('v2') + launch('gamma-grid', '', 1, 'error: config gamma-grid.yaml: missing required field seed\n') + patch('*** Begin Patch\n*** Add File: docs/runs.md\n+# Sweep runs\n+\n+Start a sweep with `labctl run alpha-sweep` and stop one with `labctl cancel `.\n*** End Patch', ['docs/runs.md']) + launch('gamma-grid', ' --seed 7', 0, 'started run g-01 (spec gamma-grid)\n', 'g-01') + spawn('alpha_monitor', 'Watch run a-01 and report when its loss stops improving.') + spawn('beta_monitor', 'Watch the beta-probe runs and compare their variants.') + spawn('gamma_monitor', 'Watch run g-01 and report any failed step.') + spawn('seed_review', 'Check whether gamma-grid seeds collide across shards.') + spawn('config_review', 'Compare the three sweep configs for unintended differences.') + w.message('user', '\n{"agent_path":"/root/schema_audit","status":"completed"}\n') + poll(90) + endTurn('Runs a-01, a-02, b-01, b-02 and g-01 are running; PR 43 is open.') + + // Turn 3: status question, two scripted launches, more helpers. + startTurn() + human("what's the status of the alpha sweep?") + exec('labctl list', 0, 'a-01 alpha-sweep running\na-02 alpha-sweep running\nb-01 beta-probe/v1 running\nb-02 beta-probe/v2 running\ng-01 gamma-grid running\n') + poll(60) + script([ + 'for (const variant of ["v3", "v4"]) {', + ' const run = await tools.exec_command({ cmd: `labctl run beta-probe --variant ${variant}` })', + ' text(run.output)', + '}', + ].join('\n'), 'started run b-03 (spec beta-probe, variant v3)\nstarted run b-04 (spec beta-probe, variant v4)\n') + runs.launched += 2 + runs.specs.add('beta-probe') + runs.betaVariants.add('v3') + runs.betaVariants.add('v4') + launchedIds.push('b-03', 'b-04') + launch('alpha-sweep', '', 0, 'started run a-03 (spec alpha-sweep)\n', 'a-03') + spawn('cache_probe', 'Measure cache hit rate of the retry lookup table.') + spawn('log_digest', 'Summarize the error lines from every running sweep.') + spawn('pr_watch', 'Watch pull requests 42 and 43 and report check results.') + poll(60) + endTurn('Alpha runs a-01, a-02 and a-03 are running; a-01 is at step 31 of 40.') + + // Turn 4: cancels, merge 42, the local graph runner. + startTurn() + human('stop launching new runs. cancel the stale alpha runs first.', true) + exec('labctl cancel a-01', 0, 'cancelled a-01\n') + exec('labctl cancel a-2', 2, 'error: no such run: a-2\n') + exec('labctl cancel a-02', 0, 'cancelled a-02\n') + exec('labctl cancel a-03', 0, 'cancelled a-03\n') + runs.cancelled += 3 + launchedIds.splice(0, launchedIds.length, ...launchedIds.filter((id) => !id.startsWith('a-'))) + exec('labctl status b-1', 2, 'error: no such run: b-1\n') + poll(45) + exec('gh pr view 42 --json reviews', 0, '{"reviews":[{"author":{"login":"review-bot"},"state":"APPROVED"}]}\n') + // The merge command backgrounds itself; only a later poll of its process shows that it succeeded. + const merged42 = exec('gh pr merge 42 --squash --delete-branch', { running: 66401 }, `Merging pull request ${REPO}#42 (feat(retry-cli): expose the budget flag)\n`) + prs.set(42, { ...prs.get(42)!, merged_at: merged42.at, reviewed_before_merge: true }) + spawn('budget_review', 'Review the budget arithmetic for off-by-one errors.') + spawn('api_diff', 'Diff the public retry API before and after the change.') + script([ + 'const patch = await tools.apply_patch(`*** Begin Patch', + '*** Add File: tools/mini-graph.mjs', + '+// Runs the sweep graph locally: topological order and per-node retries.', + '+export async function runLocalGraph(nodes, run) {', + '+ const done = new Set()', + '+ for (const node of nodes) {', + '+ for (const dep of node.after ?? []) if (!done.has(dep)) throw new Error(`unmet ${dep}`)', + '+ await run(node)', + '+ done.add(node.id)', + '+ }', + '+}', + '*** End Patch`)', + 'const check = await tools.exec_command({ cmd: "node tools/mini-graph.mjs --check" })', + 'text(patch.output + check.output)', + ].join('\n'), 'Success. Updated the following files:\nA tools/mini-graph.mjs\ngraph ok: 4 nodes\n') + changedPaths.add('tools/mini-graph.mjs') + exec('labctl status b-1', 2, 'error: no such run: b-1\n') + poll(45) + endTurn('Cancelled a-01, a-02 and a-03, merged PR 42, and added a local graph runner for the sweep.') + + // Turn 5: correction, scripted patch, merge 43 inside a verification script, the large log. + startTurn() + human("that's not what I asked for. keep the change inside the retry package.", true) + writeStdin(66401, 0, `Squashed and merged pull request ${REPO}#42 (feat(retry-cli): expose the budget flag)\nDeleted branch feat/retry-cli\n`) + script([ + 'await tools.apply_patch(`*** Begin Patch', + '*** Update File: packages/retry/src/budget.ts', + '@@', + '-export const DEFAULT_ATTEMPTS = 4', + '+export const DEFAULT_ATTEMPTS = 3', + '*** Delete File: packages/retry/src/legacy.ts', + '*** End Patch`)', + 'const test = await tools.exec_command({ cmd: "pnpm test --filter @acme/retry" })', + 'text(test.output)', + ].join('\n'), 'Test Files 6 passed (6)\n') + changedPaths.add('packages/retry/src/budget.ts') + changedPaths.add('packages/retry/src/legacy.ts') + patch('*** Begin Patch\n*** Add File: packages/retry/test/budget.test.ts\n+import { withinBudget } from \'../src/budget\'\n+\n+test(\'caps attempts\', () => expect(withinBudget(3, { max: 3 })).toBe(false))\n*** Update File: packages/retry/src/index.ts\n@@\n+export { withinBudget } from \'./budget\'\n*** End Patch', ['packages/retry/test/budget.test.ts', 'packages/retry/src/index.ts']) + spawn('bench_compare', 'Compare retry latency before and after the budget change.') + spawn('release_notes', 'Draft release notes for the retry budget change.') + poll(80) + exec('gh pr view 43 --json reviews', 0, '{"reviews":[]}\n') + // A single nested tool plus a checks command gives this script span a verification name. + const merged43 = script([ + 'const checks = await tools.exec_command({ cmd: "gh pr checks 43 --watch" })', + 'const merge = await tools.exec_command({ cmd: "gh pr merge 43 --squash" })', + 'text(checks.output + merge.output)', + ].join('\n'), `All checks were successful\nSquashed and merged pull request ${REPO}#43 (docs(retry): budget guide)\n`) + prs.set(43, { ...prs.get(43)!, merged_at: merged43.at, reviewed_before_merge: false }) + const logLines: string[] = [] + const logStart = clock.now - 3_600_000 + for (let step = 1; logLines.join('\n').length < 24 * 1024; step += 1) { + const at = new Date(logStart + step * 7_000).toISOString() + logLines.push(`${at} g-01 step=${(step % 40) + 1} shard=${step % 4} loss=${(2 + random()).toFixed(4)} seed=7`) + } + const lastLogLine = 'g-01 finished: status=failed step=40 reason=seed collision on shard 3' + logLines.push(lastLogLine) + const logOutputLine = exec('labctl logs g-01', 0, `${logLines.join('\n')}\n`).output.line + endTurn('PR 43 is merged. Run g-01 failed at step 40.') + + // Turns 6 to 8: praise, a substantive question, then a short follow-up. + startTurn() + human('thanks, that looks right') + poll(60) + endTurn('Thanks. The beta-probe runs are still going.') + startTurn() + human('why does the gamma grid keep failing at step 40? is it the seed or the config?') + poll(50) + endTurn('Looking at the g-01 log now.') + startTurn() + human('ya?') + poll(STATUS_POLLS - statusPolls) + endTurn('The g-01 log ends with a seed collision on shard 3, so it is the seed, not the config.') + tokenCount() + // Codex writes a repeated token count with an unchanged cumulative total; summing deltas would double it. + w.row('event_msg', lastTokenEvent!) + w.message('user', '\n{"agent_path":"/root/release_notes","status":"completed"}\n') + const last = w.message('user', '\n /work/orbit/packages/retry\n') + + if (!child) throw new Error('operator plan did not spawn the forked child') + const last2 = humans.at(-2)! + const lastHuman = humans.at(-1)! + const gold: GoldAnswers = { + 'op.subagents': { + spawn_calls: spawns.length, + failed_spawns: spawns.filter((item) => item.failed).length, + task_names: [...new Set(spawns.filter((item) => !item.failed).map((item) => item.name))].sort(), + }, + 'op.pull-requests': { + prs: [...prs.entries()].sort(([a], [b]) => a - b).map(([number, pr]) => ({ number, ...pr })), + }, + 'op.runs': { + launched: runs.launched, + failed_launches: runs.failed, + cancelled: runs.cancelled, + specs: [...runs.specs].sort(), + beta_probe_variants: [...runs.betaVariants].sort(), + }, + 'op.last-human-turn': { + last: quote(file, lastHuman, lastHuman.text), + last_at: lastHuman.at, + previous: quote(file, last2, last2.text), + }, + 'op.role': { + role: 'operator', + merged_prs: [...prs.values()].filter((pr) => pr.merged_at).length, + launched_runs: runs.launched, + spawn_calls: spawns.length, + }, + 'op.local-copy': { path: 'tools/mini-graph.mjs' }, + 'op.corrections': { corrections: corrections.map((item) => quote(file, item, item.text)) }, + 'op.time-bounds': { first_record_at: first.at, last_record_at: last.at }, + 'op.exit-codes': { nonzero_exec_commands: exits.length, codes: [...new Set(exits)].sort((a, b) => a - b) }, + 'op.changed-files': { paths: [...changedPaths].sort() }, + 'op.status-polls': { status_commands: statusPolls }, + 'op.large-output': { last_line: quote(file, { line: logOutputLine, at: '' }, lastLogLine) }, + 'op.tokens': { + input_tokens: cumulative.input_tokens, + cached_input_tokens: cumulative.cached_input_tokens, + output_tokens: cumulative.output_tokens, + }, + } + return { file, sessionId, rows: w.out.rows, first, last, child, gold } +} + +function childSession(random: Random, parent: OperatorPlan): { file: string; content: string; gold: GoldAnswers } { + const { sessionId, agentPath, forkRows, spawnAtMs } = parent.child + const clock = new Clock(spawnAtMs + 1_000, random) + const w = new CodexWriter(clock, random) + const file = `codex/sessions/2026/03/14/rollout-${new Date(spawnAtMs + 1_000).toISOString().slice(0, 19).replaceAll(':', '-')}-${sessionId}.jsonl` + const forkAt = new Date(spawnAtMs + 1_000).toISOString() + w.row('session_meta', { + id: sessionId, + timestamp: forkAt, + cwd: OPERATOR_CWD, + originator: 'codex_cli_rs', + cli_version: '0.150.0', + parent_thread_id: parent.sessionId, + thread_source: 'subagent', + agent_nickname: 'Agent-flak', + agent_path: agentPath, + source: { subagent: { thread_spawn: { parent_thread_id: parent.sessionId, depth: 1, agent_path: agentPath, agent_nickname: 'Agent-flak' } } }, + }, forkAt) + // Inherited history keeps its content but carries the fork time, as Codex rewrites it. + for (const row of forkRows) { + const parsed = JSON.parse(row) as Record + w.out.push({ ...parsed, timestamp: forkAt }) + } + const startedAt = clock.next() + const started = w.row('event_msg', { type: 'task_started', turn_id: sessionId, started_at: Math.floor(Date.parse(startedAt) / 1_000), model_context_window: 272_000 }, startedAt) + w.row('turn_context', { cwd: OPERATOR_CWD, approval_policy: 'never', sandbox_policy: { mode: 'workspace-write' }, model: CODEX_MODEL }) + w.message('user', 'Find which retry tests fail intermittently and why.') + let ownTools = 0 + let failed = 0 + const exec = (cmd: string, exit: number, body: string): void => { + const call = w.call('exec_command', { cmd, workdir: OPERATOR_CWD, yield_time_ms: 10_000 }) + w.output(call, shellOutput(random, exit, body)) + ownTools += 1 + if (exit !== 0) failed += 1 + } + exec('pnpm vitest run packages/retry --reporter dot', 1, 'FAIL packages/retry/test/jitter.test.ts > spreads retries\n expected 120 to be less than 100\n') + exec('pnpm vitest run packages/retry -t "spreads retries" --repeat 20', 0, '20 passed, 0 failed\n') + exec('git log -3 --oneline -- packages/retry', 0, '9f1c2ab feat(retry): add jitter\n4d0e7aa test(retry): cover jitter bounds\n') + const patchText = '*** Begin Patch\n*** Update File: packages/retry/test/jitter.test.ts\n@@\n- const random = Math.random\n+ const random = seeded(42)\n*** End Patch' + const patchCall = w.custom('apply_patch', patchText) + w.output(patchCall, JSON.stringify({ output: patchSummary(patchText), metadata: { exit_code: 0, duration_seconds: 0.1 } })) + ownTools += 1 + exec('pnpm vitest run packages/retry', 0, 'Test Files 7 passed (7)\n') + const send = w.call('send_message', { target: parent.sessionId, message: 'jitter.test.ts used an unseeded random source; seeded it with 42.' }) + w.output(send, JSON.stringify({ delivered: true })) + ownTools += 1 + w.message('assistant', 'The flaky test was jitter.test.ts: it used an unseeded random source. It now uses a seeded one.') + w.row('event_msg', { type: 'token_count', info: { last_token_usage: { input_tokens: 18_000, cached_input_tokens: 15_000, output_tokens: 900, reasoning_output_tokens: 300, total_tokens: 18_900 }, total_token_usage: { input_tokens: 18_000, cached_input_tokens: 15_000, output_tokens: 900, reasoning_output_tokens: 300, total_tokens: 18_900 }, model_context_window: 272_000 } }) + w.row('event_msg', { type: 'task_complete', turn_id: sessionId, last_agent_message: 'Seeded the jitter test.' }) + return { + file, + content: w.out.text(), + gold: { + 'child.lineage': { parent_session_id: parent.sessionId, agent_path: agentPath }, + 'child.own-work': { task_started_at: started.at, own_tool_calls: ownTools, failed_commands: failed }, + 'child.spawned': { spawned_session_ids: [] }, + }, + } +} + +interface ClaudeTool { + name: string + input: Record + result: string | Array<{ type: 'text'; text: string }> + isError?: boolean +} + +class ClaudeWriter { + readonly out = new Jsonl() + private parent: string | null = null + + constructor( + private readonly clock: Clock, + private readonly random: Random, + private readonly sessionId: string, + private readonly sidechain: { agentId: string } | undefined, + ) {} + + private base(type: 'user' | 'assistant', at: string): Record { + const uuid = uuidV4(this.random) + const row = { + parentUuid: this.parent, + isSidechain: this.sidechain !== undefined, + userType: 'external', + cwd: CLAUDE_CWD, + sessionId: this.sessionId, + version: '2.1.0', + ...(this.sidechain ? { agentId: this.sidechain.agentId } : {}), + type, + uuid, + timestamp: at, + } + this.parent = uuid + return row + } + + user(content: string): Placed { + const at = this.clock.next() + return { line: this.out.push({ ...this.base('user', at), message: { role: 'user', content } }), at } + } + + assistant(text: string | null, tools: Array<{ id: string; name: string; input: Record }> = []): Placed { + const at = this.clock.next() + const content = [ + ...(text ? [{ type: 'text', text }] : []), + ...tools.map((tool) => ({ type: 'tool_use', id: tool.id, name: tool.name, input: tool.input })), + ] + return { + line: this.out.push({ + ...this.base('assistant', at), + message: { + id: `msg_${base62(this.random, 24)}`, + type: 'message', + role: 'assistant', + model: CLAUDE_MODEL, + content, + usage: { input_tokens: 40 + Math.floor(this.random() * 200), cache_read_input_tokens: 12_000, output_tokens: 80 + Math.floor(this.random() * 400) }, + }, + requestId: `req_${base62(this.random, 24)}`, + }), + at, + } + } + + results(items: Array<{ id: string; tool: ClaudeTool }>): Placed { + const at = this.clock.next() + return { + line: this.out.push({ + ...this.base('user', at), + message: { + role: 'user', + content: items.map(({ id, tool }) => ({ + tool_use_id: id, + type: 'tool_result', + content: tool.result, + ...(tool.isError ? { is_error: true } : {}), + })), + }, + }), + at, + } + } + + /** One assistant tool call followed by its result record. */ + tool(tool: ClaudeTool, text: string | null = null): Placed { + const id = `toolu_${base62(this.random, 24)}` + this.assistant(text, [{ id, name: tool.name, input: tool.input }]) + return this.results([{ id, tool }]) + } +} + +function claudeSession(random: Random): { files: BenchFile[]; session: BenchSession; gold: GoldAnswers } { + const clock = new Clock(CLAUDE_START_MS, random) + const sessionId = uuidV4(random) + const dir = 'claude/projects/-work-ledger' + const file = `${dir}/${sessionId}.jsonl` + const w = new ClaudeWriter(clock, random, sessionId, undefined) + let bashCalls = 0 + let bashErrors = 0 + let subagentBashErrors = 0 + const bash = (writer: ClaudeWriter, command: string, result: string, isError = false, inSubagent = false): Placed => { + bashCalls += 1 + if (isError) { + bashErrors += 1 + if (inSubagent) subagentBashErrors += 1 + } + return writer.tool({ name: 'Bash', input: { command, description: `Run ${command.split(' ')[0]}` }, result, isError }) + } + + w.user('The nightly import job fails on the March CSV. Find the cause and fix it.') + const firstErrorText = 'Exit code 1\nFAIL src/import.test.ts > parses the March file\nRangeError: Invalid time value' + const firstError = bash(w, 'pnpm test --filter import', firstErrorText, true) + bash(w, 'git log --oneline -5 -- src/import', 'a1b2c3d fix(import): trim byte order mark\n7e8f9a0 feat(import): stream rows') + w.tool({ name: 'Read', input: { file_path: '/work/ledger/src/dates.ts' }, result: 'File does not exist.', isError: true }) + + const tasks = [ + { description: 'Map the import pipeline', subagent_type: 'Explore', prompt: 'List every module the nightly import passes a row through, in order.' }, + { description: 'Reproduce the date failure', subagent_type: 'general-purpose', prompt: 'Reproduce the RangeError with the smallest CSV row you can find.' }, + { description: 'Survey date parsing', subagent_type: 'Explore', prompt: 'Find every place in src/ that parses a date string.' }, + ] + const taskIds = tasks.map(() => `toolu_${base62(random, 24)}`) + w.assistant('Splitting the investigation across three subagents.', tasks.map((input, index) => ({ id: taskIds[index]!, name: 'Task', input }))) + + const subagentFiles: BenchFile[] = [] + const summaries: string[] = [] + const subagentScripts: Array> = [ + [ + { command: 'rg -l "importRow" src', result: 'src/import.ts\nsrc/rows.ts\nsrc/parse-date.ts' }, + { command: 'sed -n 1,40p src/import.ts', result: 'export async function importFile(path) {\n for await (const row of rows(path)) await importRow(row)\n}' }, + ], + [ + { command: 'node scripts/import-one.mjs fixtures/march.csv', result: 'Exit code 1\nRangeError: Invalid time value at parseDate (src/parse-date.ts:18)', isError: true }, + { command: 'grep -n "2026-02-30" fixtures/march.csv', result: '412:2026-02-30,ACME,12.00' }, + ], + [ + { command: 'rg -n "new Date\\(" src', result: 'src/parse-date.ts:18: return new Date(value).toISOString()' }, + ], + ] + let subagentsDoneAt = clock.now + for (const [index, task] of tasks.entries()) { + const agentId = hex(random, 17) + const subClock = new Clock(clock.now + index * 1_500, random) + const sub = new ClaudeWriter(subClock, random, sessionId, { agentId }) + sub.user(task.prompt) + for (const step of subagentScripts[index]!) bash(sub, step.command, step.result, step.isError ?? false, true) + const summary = [ + 'The row path is importFile, then rows, then importRow, then parseDate.', + 'Row 412 of fixtures/march.csv holds the impossible date 2026-02-30, and parseDate throws on it.', + 'Only src/parse-date.ts parses date strings, at line 18.', + ][index]! + sub.assistant(summary) + subagentsDoneAt = Math.max(subagentsDoneAt, subClock.now) + summaries.push(summary) + const subPath = `${dir}/${sessionId}/subagents/agent-${agentId}.jsonl` + subagentFiles.push({ path: subPath, content: sub.out.text() }) + subagentFiles.push({ path: subPath.replace(/\.jsonl$/, '.meta.json'), content: `${JSON.stringify({ agentType: task.subagent_type, toolUseId: taskIds[index] })}\n` }) + } + // The Task results arrive after every child has finished. + clock.passAt(subagentsDoneAt) + w.results(tasks.map((_, index) => ({ id: taskIds[index]!, tool: { name: 'Task', input: {}, result: [{ type: 'text', text: summaries[index]! }] } }))) + bash(w, 'pnpm test --filter import -- -t "March"', 'Exit code 1\nFAIL src/import.test.ts > rejects 2026-02-30\nexpected parseDate to return null', true) + w.tool({ name: 'Edit', input: { file_path: '/work/ledger/src/parse-date.ts', old_string: 'return new Date(value).toISOString()', new_string: 'const date = new Date(value)\n return Number.isNaN(date.getTime()) ? null : date.toISOString()' }, result: 'The file has been updated.' }) + bash(w, 'pnpm test --filter import', 'Test Files 4 passed (4)') + w.assistant('Fixed: parseDate now returns null for impossible dates such as 2026-02-30 instead of throwing.') + + return { + files: [{ path: file, content: w.out.text() }, ...subagentFiles], + session: { id: 'claude', harness: 'claude-code', path: file, sessionId }, + gold: { + 'claude.tasks': { + task_calls: tasks.length, + subagent_types: [...new Set(tasks.map((task) => task.subagent_type))].sort(), + descriptions: tasks.map((task) => task.description).sort(), + }, + 'claude.bash': { bash_calls: bashCalls, failed_bash_calls: bashErrors, failed_in_subagents: subagentBashErrors }, + 'claude.first-bash-error': { output: quote(file, firstError, firstErrorText) }, + }, + } +} + +/** Build every fixture file and its gold in memory. The same seed always yields the same bytes. */ +export function generateBench(): GeneratedBench { + const random = mulberry32(SEED) + const operator = operatorSession(random) + const child = childSession(random, operator) + const claude = claudeSession(random) + const files: BenchFile[] = [ + { path: operator.file, content: `${operator.rows.join('\n')}\n` }, + { path: child.file, content: child.content }, + ...claude.files, + ].sort((a, b) => a.path.localeCompare(b.path)) + const basenames = new Set(files.map((item) => item.path.split('/').at(-1))) + if (basenames.size !== files.length) throw new Error('fixture file basenames must be unique so line citations resolve') + return { + manifest: { + version: 1, + sessions: [ + { id: 'codex-operator', harness: 'codex', path: operator.file, sessionId: operator.sessionId }, + { id: 'codex-child', harness: 'codex', path: child.file, sessionId: operator.child.sessionId }, + claude.session, + ], + files: files.map((item) => item.path), + }, + files, + gold: { ...operator.gold, ...child.gold, ...claude.gold }, + } +} + +/** Write the fixture tree under `root`. Arms read only this tree, never the gold. */ +export async function writeFixtures(root: string, bench: GeneratedBench = generateBench()): Promise { + for (const item of bench.files) { + const path = join(root, item.path) + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, item.content) + } + return bench +} diff --git a/bench/audit/questions.ts b/bench/audit/questions.ts new file mode 100644 index 0000000..6bfda32 --- /dev/null +++ b/bench/audit/questions.ts @@ -0,0 +1,308 @@ +/** + * The audit questions, each with a fixed answer schema. + * + * A schema field names both the JSON type an arm must return and the exact rule + * the scorer applies to it, so the prompt an arm sees and the scorer never + * disagree. `probes` lists the failure classes from the improvement brief that + * the question exercises (F1 ordered user turns, F2 enumeration, F3 timestamps, + * F4 links across spans, F5 changed files, F6 verbatim tool output, F9 + * truncation, F11 derived fields). + */ + +import type { BenchManifest, BenchSessionId } from './fixtures.js' + +/** One scored field. Every leaf also accepts null, which means "not in trace". */ +export type Field = + | { kind: 'count' } + | { kind: 'string'; enum?: readonly string[] } + | { kind: 'boolean' } + /** ISO 8601 time, correct within 1 s. */ + | { kind: 'time' } + /** Unordered collection, compared as a set. */ + | { kind: 'set'; of: 'integer' | 'string' } + /** A verbatim quote with a citation of the record it came from. */ + | { kind: 'quote' } + /** An unordered collection of quotes. */ + | { kind: 'quotes' } + /** Objects matched by `key`; each other field is scored per object. */ + | { kind: 'records'; key: string; keyOf: 'integer' | 'string'; fields: Readonly> } + +export type AnswerSchema = Readonly> + +export interface Question { + id: string + session: BenchSessionId + probes: readonly string[] + text: string + /** Held-out rewordings. Arms answer them with the same schema. */ + paraphrases: readonly string[] + schema: AnswerSchema +} + +const count = { kind: 'count' } as const +const time = { kind: 'time' } as const +const quoteField = { kind: 'quote' } as const + +export const QUESTIONS: readonly Question[] = [ + { + id: 'op.subagents', + session: 'codex-operator', + probes: ['F2'], + text: 'How many spawn_agent calls did the session make, how many of them failed, and what task names did the successful spawns receive? Give each task name exactly as written in the task_name argument.', + paraphrases: ['Count every native subagent spawn attempt in this session and the failed attempts among them, and list the task_name argument of each spawn that succeeded, verbatim.'], + schema: { spawn_calls: count, failed_spawns: count, task_names: { kind: 'set', of: 'string' } }, + }, + { + id: 'op.pull-requests', + session: 'codex-operator', + probes: ['F2', 'F3', 'F4'], + text: 'Which pull requests did the session create? For each one, give its number, the time of the tool call that issued the create command, the time of the tool call that issued the successful merge command, and whether the session was shown a review of it before that merge.', + paraphrases: ['List every pull request this session opened. For each, report the PR number, when the session opened it and when it merged it (use the times of the tool calls that ran those commands), and whether any review of it was visible to the session before the merge.'], + schema: { + prs: { + kind: 'records', + key: 'number', + keyOf: 'integer', + fields: { created_at: time, merged_at: time, reviewed_before_merge: { kind: 'boolean' } }, + }, + }, + }, + { + id: 'op.runs', + session: 'codex-operator', + probes: ['F2'], + text: 'How many runs did the session start successfully with `labctl run`, how many `labctl run` attempts failed, how many runs did it cancel successfully with `labctl cancel`, which specs did it start, and which --variant values did it start for the beta-probe spec?', + paraphrases: ['Count the successful `labctl run` launches, the failed launch attempts, and the successful `labctl cancel` commands in this session. Also list the specs it launched and every beta-probe variant it launched.'], + schema: { + launched: count, + failed_launches: count, + cancelled: count, + specs: { kind: 'set', of: 'string' }, + beta_probe_variants: { kind: 'set', of: 'string' }, + }, + }, + { + id: 'op.last-human-turn', + session: 'codex-operator', + probes: ['F1'], + text: 'What was the last message the human typed in this session, and when was it sent? Also quote the human message immediately before it. Text the harness injected into the conversation is not a human message.', + paraphrases: ['Quote the final human-written message of this session with its timestamp, and quote the human-written message that preceded it. Ignore anything the harness inserted.'], + schema: { last: quoteField, last_at: time, previous: quoteField }, + }, + { + id: 'op.role', + session: 'codex-operator', + probes: ['F2'], + text: 'Did this session act as an operator (it launched, changed, and merged work itself) or as an observer (it only read and steered)? Give the role, the number of pull requests it merged, the number of runs it started successfully, and the number of spawn_agent calls it made.', + paraphrases: ['Classify this session as operator or observer, and support the classification with three counts: pull requests it merged, runs it launched successfully, and spawn_agent calls.'], + schema: { + role: { kind: 'string', enum: ['operator', 'observer'] }, + merged_prs: count, + launched_runs: count, + spawn_calls: count, + }, + }, + { + id: 'op.local-copy', + session: 'codex-operator', + probes: ['F5'], + text: 'The shared package @acme/runtime already exports runGraph. Which file did the session add that implements its own graph runner instead of importing runGraph? Give the path as written in the patch.', + paraphrases: ['This session re-implemented a capability that @acme/runtime already provides as runGraph. Name the file it created for that local version, using the path from the patch.'], + schema: { path: { kind: 'string' } }, + }, + { + id: 'op.corrections', + session: 'codex-operator', + probes: ['F1'], + text: 'Quote every human message in which the human corrects the agent, meaning the human tells it to stop, to wait, or to do something differently from what it did.', + paraphrases: ['Find each message where the human pushed back on the agent (told it to stop, hold off, or change course) and quote each one.'], + schema: { corrections: { kind: 'quotes' } }, + }, + { + id: 'op.time-bounds', + session: 'codex-operator', + probes: ['F3'], + text: 'When were the first and the last records of this session written?', + paraphrases: ['Give the timestamps of the earliest and the latest records in this session.'], + schema: { first_record_at: time, last_record_at: time }, + }, + { + id: 'op.exit-codes', + session: 'codex-operator', + probes: ['F2', 'F11'], + text: 'How many direct exec_command tool calls (not commands run inside exec scripts) exited with a nonzero exit code, and which distinct nonzero exit codes occurred?', + paraphrases: ['Among exec_command calls made directly by the agent, excluding those inside exec scripts, count the ones whose process exited nonzero and list the distinct nonzero codes.'], + schema: { nonzero_exec_commands: count, codes: { kind: 'set', of: 'integer' } }, + }, + { + id: 'op.changed-files', + session: 'codex-operator', + probes: ['F5'], + text: 'List every file path that any apply_patch call in this session added, updated, or deleted.', + paraphrases: ['Which files did this session change through apply_patch? Include files it created, edited, and deleted.'], + schema: { paths: { kind: 'set', of: 'string' } }, + }, + { + id: 'op.status-polls', + session: 'codex-operator', + probes: ['F2'], + text: 'How many times did the session run `labctl status`?', + paraphrases: ['Count the `labctl status` commands this session executed.'], + schema: { status_commands: count }, + }, + { + id: 'op.large-output', + session: 'codex-operator', + probes: ['F9'], + text: 'Quote the last line of output from the session\'s `labctl logs` command.', + paraphrases: ['What is the final line printed by the `labctl logs` command in this session? Quote it.'], + schema: { last_line: quoteField }, + }, + { + id: 'op.tokens', + session: 'codex-operator', + probes: ['F11'], + text: 'What were the session\'s final cumulative input tokens, cached input tokens, and output tokens, as its last token count reported them?', + paraphrases: ['Report the running totals of input, cached input, and output tokens at the end of this session.'], + schema: { input_tokens: count, cached_input_tokens: count, output_tokens: count }, + }, + { + id: 'child.lineage', + session: 'codex-child', + probes: ['F4'], + text: 'Which session spawned this one, and what agent path was this session given?', + paraphrases: ['Name the parent session id of this session and the agent path it runs under.'], + schema: { parent_session_id: { kind: 'string' }, agent_path: { kind: 'string' } }, + }, + { + id: 'child.own-work', + session: 'codex-child', + probes: ['F2', 'F3'], + text: 'Counting only work this session did itself, not history it inherited from its parent: when did its own task start, how many tool calls did it make, and how many of its exec_command calls exited with a nonzero code?', + paraphrases: ['Leave out any history this session inherited from its parent. When did its own task begin, how many tool calls did it issue, and how many of its exec_command calls failed with a nonzero exit code?'], + schema: { task_started_at: time, own_tool_calls: count, failed_commands: count }, + }, + { + id: 'child.spawned', + session: 'codex-child', + probes: ['F11'], + text: 'Which sessions did this session spawn? Give their session ids, or an empty list if it spawned none.', + paraphrases: ['List the ids of every child session this session created; answer with an empty list when there are none.'], + schema: { spawned_session_ids: { kind: 'set', of: 'string' } }, + }, + { + id: 'claude.tasks', + session: 'claude', + probes: ['F2'], + text: 'How many Task tool calls did the main session make, which subagent_type values did they use, and what were their descriptions?', + paraphrases: ['Count the Task subagent launches in the main transcript, and list the distinct subagent types and the description of each launch.'], + schema: { task_calls: count, subagent_types: { kind: 'set', of: 'string' }, descriptions: { kind: 'set', of: 'string' } }, + }, + { + id: 'claude.bash', + session: 'claude', + probes: ['F2'], + text: 'Across the main session and its subagents, how many Bash tool calls were made, how many returned an error result, and how many of those errors happened inside subagents?', + paraphrases: ['Including subagent transcripts, count all Bash calls, the Bash calls whose result was an error, and the subset of those errors that came from subagents.'], + schema: { bash_calls: count, failed_bash_calls: count, failed_in_subagents: count }, + }, + { + id: 'claude.first-bash-error', + session: 'claude', + probes: ['F6'], + text: 'Quote the full output of the first Bash call in the main session that returned an error.', + paraphrases: ['In the main transcript, find the earliest Bash call whose result was an error and quote its output in full.'], + schema: { output: quoteField }, + }, +] + +export function questionById(id: string): Question | undefined { + return QUESTIONS.find((question) => question.id === id) +} + +const nullable = (type: string): { type: [string, 'null'] } => ({ type: [type, 'null'] }) + +function fieldJsonSchema(field: Field): Record { + switch (field.kind) { + case 'count': return { ...nullable('integer'), minimum: 0 } + case 'string': return field.enum ? { enum: [...field.enum, null] } : nullable('string') + case 'boolean': return nullable('boolean') + case 'time': return { ...nullable('string'), format: 'date-time' } + case 'set': return { ...nullable('array'), items: { type: field.of }, uniqueItems: true } + case 'quote': return QUOTE_SCHEMA + case 'quotes': return { ...nullable('array'), items: QUOTE_SCHEMA } + case 'records': return { + ...nullable('array'), + items: objectSchema({ [field.key]: { kind: field.keyOf === 'integer' ? 'count' : 'string' }, ...field.fields }), + } + } +} + +const QUOTE_SCHEMA = { + type: ['object', 'null'], + additionalProperties: false, + required: ['text', 'cite'], + properties: { + text: { type: 'string', description: 'Verbatim text from the session.' }, + cite: { type: 'string', description: 'A span id, a trace:///span/ URI, or : of the source record.' }, + }, +} as const + +function objectSchema(fields: AnswerSchema): Record { + return { + type: 'object', + additionalProperties: false, + required: Object.keys(fields), + properties: Object.fromEntries(Object.entries(fields).map(([name, field]) => [name, fieldJsonSchema(field)])), + } +} + +/** The JSON Schema (2020-12 subset) an arm's answer to this question must match. */ +export function answerJsonSchema(question: Question): Record { + return { $schema: 'https://json-schema.org/draft/2020-12/schema', ...objectSchema(question.schema) } +} + +export const ANSWER_RULES = [ + 'Answer with one JSON object that matches the schema. Put nothing else in the answer.', + 'Use null for a value the session does not contain.', + 'Give times in ISO 8601 UTC. A time is correct within 1 second.', + 'Counts, numbers, names, and paths are scored exactly.', + 'A quote is {"text", "cite"}. The text must be verbatim. The cite must name the record the text came from: a span id, a trace:///span/ URI, or : of the source record, with the file path relative to the fixture root and the line 1-based.', +] as const + +export interface PromptRow { + question: string + variant: number + heldOut: boolean + session: string + harness: string + sessionPath: string + prompt: string + schema: Record +} + +/** Every question and paraphrase as the exact text an arm receives. Variant 0 is the canonical wording. */ +export function promptRows(manifest: BenchManifest): PromptRow[] { + return QUESTIONS.flatMap((question) => { + const session = manifest.sessions.find((item) => item.id === question.session) + if (!session) throw new Error(`manifest has no session ${question.session}`) + const schema = answerJsonSchema(question) + return [question.text, ...question.paraphrases].map((text, variant) => ({ + question: question.id, + variant, + heldOut: variant > 0, + session: session.sessionId, + harness: session.harness, + sessionPath: session.path, + prompt: [ + `Session: ${session.path} (${session.harness}, session id ${session.sessionId}).`, + `Question: ${text}`, + 'Rules:', + ...ANSWER_RULES.map((rule) => `- ${rule}`), + 'Answer JSON Schema:', + JSON.stringify(schema, null, 2), + ].join('\n'), + schema, + })) + }) +} diff --git a/bench/audit/score.test.ts b/bench/audit/score.test.ts new file mode 100644 index 0000000..383cdd6 --- /dev/null +++ b/bench/audit/score.test.ts @@ -0,0 +1,241 @@ +/** + * Scorer tests. Nothing here calls a model: the grader is exact match, so its + * behavior is fully testable, and these tests are what keep it honest. + * + * The first test doubles as the gold's own consistency check. Submitting the + * gold as an arm's answers must score every question correct through the same + * path a real arm takes, including citation resolution through the traces + * adapters. If it does not, the answer key contradicts the fixture bytes. + */ + +import { execFile } from 'node:child_process' +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { type CitationIndex, loadCitationIndex, spanRecordMap } from './citations.js' +import { generateBench, writeFixtures } from './fixtures.js' +import { QUESTIONS, questionById } from './questions.js' +import { type AnswerRow, parseAnswersFile, renderArmScore, scoreAnswer, scoreArm } from './score.js' + +const execFileAsync = promisify(execFile) +const tsx = join(process.cwd(), 'node_modules', 'tsx', 'dist', 'loader.mjs') +const cli = join(process.cwd(), 'bench', 'audit', 'cli.ts') + +const bench = generateBench() +let root = '' +let index: CitationIndex + +beforeAll(async () => { + root = await mkdtemp(join(tmpdir(), 'traces-bench-score-')) + await writeFixtures(root, bench) + index = await loadCitationIndex(root, bench.manifest) +}) + +afterAll(async () => { + if (root) await rm(root, { recursive: true, force: true }) +}) + +/** The gold answer for one question, deep-copied so a test can spoil one field. */ +const goldAnswer = (id: string): Record => structuredClone(bench.gold[id]!) +const score = (id: string, answer: unknown): ReturnType => + scoreAnswer(questionById(id)!, { question: id, answer }, bench.gold, index) + +describe('the gold scores itself correct', () => { + it.each(QUESTIONS.map((question) => question.id))('%s', (id) => { + const scored = score(id, goldAnswer(id)) + expect(scored.verdict).toBe('correct') + expect(scored.leaves.every((leaf) => leaf.correct)).toBe(true) + for (const citation of scored.leaves.flatMap((leaf) => leaf.citations)) { + expect(citation).toEqual({ cite: citation.cite, resolved: true, verified: true, onGold: true }) + } + }) +}) + +describe('exact-match leaves', () => { + it('marks a wrong count wrong and a mixed answer partial', () => { + expect(score('op.status-polls', { status_commands: 522 }).verdict).toBe('wrong') + const mixed = goldAnswer('op.subagents') + mixed.failed_spawns = 0 + expect(score('op.subagents', mixed).verdict).toBe('partial') + }) + + it('rejects a count that is not an integer', () => { + expect(score('op.status-polls', { status_commands: '523' }).verdict).toBe('wrong') + expect(score('op.status-polls', { status_commands: 523.5 }).verdict).toBe('wrong') + }) + + it('accepts a time within one second and rejects one beyond it', () => { + const gold = bench.gold['op.time-bounds'] as { first_record_at: string; last_record_at: string } + const shift = (iso: string, ms: number): string => new Date(Date.parse(iso) + ms).toISOString() + expect(score('op.time-bounds', { + first_record_at: shift(gold.first_record_at, 900), + last_record_at: shift(gold.last_record_at, -1_000), + }).verdict).toBe('correct') + expect(score('op.time-bounds', { + first_record_at: shift(gold.first_record_at, 1_001), + last_record_at: gold.last_record_at, + }).verdict).toBe('partial') + }) + + it('compares sets without order and without tolerating a gap', () => { + const answer = goldAnswer('op.runs') + answer.specs = [...(answer.specs as string[])].reverse() + expect(score('op.runs', answer).verdict).toBe('correct') + answer.specs = (answer.specs as string[]).slice(1) + expect(score('op.runs', answer).verdict).toBe('partial') + }) + + it('records a null where the gold has a value as a false "not in trace"', () => { + const scored = score('op.local-copy', { path: null }) + expect(scored.verdict).toBe('wrong') + expect(scored.leaves[0]!.falseNotInTrace).toBe(true) + expect(score('op.local-copy', { path: 'tools/mini-graph.mjs' }).leaves[0]!.falseNotInTrace).toBe(false) + }) + + it('scores an empty list as the right answer only where the gold is empty', () => { + expect(score('child.spawned', { spawned_session_ids: [] }).verdict).toBe('correct') + expect(score('child.spawned', { spawned_session_ids: ['made-up'] }).verdict).toBe('wrong') + }) + + it('matches records by key and fails the ones it cannot find', () => { + const answer = goldAnswer('op.pull-requests') + const prs = answer.prs as Array> + answer.prs = prs.slice(0, 2) + const scored = score('op.pull-requests', answer) + expect(scored.verdict).toBe('partial') + expect(scored.leaves.filter((leaf) => leaf.path.startsWith('prs[43]')).every((leaf) => !leaf.correct)).toBe(true) + expect(scored.leaves.find((leaf) => leaf.path === 'prs[*].number')!.correct).toBe(false) + }) + + it('fails a merge time taken from the wrong tool call', () => { + const answer = goldAnswer('op.pull-requests') + const prs = answer.prs as Array> + prs[1]!.merged_at = prs[2]!.merged_at + const scored = score('op.pull-requests', answer) + expect(scored.verdict).toBe('partial') + expect(scored.leaves.find((leaf) => leaf.path === 'prs[42].merged_at')!.correct).toBe(false) + }) +}) + +describe('quotes and citations', () => { + const goldQuote = bench.gold['op.large-output']!.last_line as { text: string; cite: string } + + it('needs the text verbatim and the citation on the gold record', () => { + expect(score('op.large-output', { last_line: goldQuote }).verdict).toBe('correct') + expect(score('op.large-output', { last_line: { text: 'g-01 failed', cite: goldQuote.cite } }).verdict).toBe('wrong') + }) + + it('rejects a real citation that names another record, and says so', () => { + const other = bench.gold['op.corrections']!.corrections as Array<{ cite: string }> + const scored = score('op.large-output', { last_line: { text: goldQuote.text, cite: other[0]!.cite } }) + expect(scored.verdict).toBe('wrong') + expect(scored.leaves[0]!.citations[0]).toMatchObject({ resolved: true, verified: false, onGold: false }) + }) + + it('reports a citation that names no known record', () => { + const scored = score('op.large-output', { last_line: { text: goldQuote.text, cite: 'nowhere.jsonl:12' } }) + expect(scored.leaves[0]!.citations[0]).toMatchObject({ resolved: false, verified: false, onGold: false }) + }) + + it('accepts a span id whose span came from the gold record', async () => { + const spans = await spanRecordMap(root, bench.manifest) + const [file, line] = [goldQuote.cite.slice(0, goldQuote.cite.lastIndexOf(':')), Number(goldQuote.cite.split(':').at(-1))] + const spanId = [...spans].find(([, refs]) => refs.some((ref) => ref.file === file && ref.line === line))?.[0] + expect(spanId).toBeDefined() + const scored = score('op.large-output', { last_line: { text: goldQuote.text, cite: spanId! } }) + expect(scored.verdict).toBe('correct') + expect(scored.leaves[0]!.citations[0]).toMatchObject({ resolved: true, onGold: true }) + }) + + it('needs every correction quoted once, and no invented one', () => { + const gold = goldAnswer('op.corrections') + const corrections = gold.corrections as Array<{ text: string; cite: string }> + expect(score('op.corrections', { corrections: [...corrections].reverse() }).verdict).toBe('correct') + expect(score('op.corrections', { corrections: corrections.slice(1) }).verdict).toBe('wrong') + expect(score('op.corrections', { + corrections: [...corrections, { text: corrections[0]!.text, cite: corrections[0]!.cite }], + }).verdict).toBe('wrong') + }) +}) + +describe('answers files', () => { + it('rejects every problem at once', () => { + expect(() => parseAnswersFile({ arm: '', answers: [{ question: 'op.nope', answer: null }, { question: 'op.runs', variant: 9 }] })) + .toThrow(/arm must be a non-empty string[\s\S]*not a known question id[\s\S]*variant must be 0 to[\s\S]*answer is missing/) + }) + + it('accepts a well-formed file', () => { + const file = parseAnswersFile({ arm: 'baseline', answers: [{ question: 'op.runs', answer: null, cost_usd: 0.01, cost_basis: 'estimated' }] }) + expect(file.arm).toBe('baseline') + }) +}) + +describe('arm scores', () => { + const answers = (rows: readonly AnswerRow[]) => scoreArm({ arm: 'test-arm', answers: [...rows] }, bench.gold, index) + + it('separates canonical wordings from held-out paraphrases and lists what was skipped', () => { + const scored = answers([ + { question: 'op.status-polls', variant: 0, answer: goldAnswer('op.status-polls'), wall_ms: 100, cost_usd: 0.5, cost_basis: 'observed' }, + { question: 'op.status-polls', variant: 1, answer: { status_commands: 1 }, wall_ms: 300 }, + ]) + expect(scored.canonical).toMatchObject({ attempts: 1, correct: 1, wrong: 0 }) + expect(scored.heldOut).toMatchObject({ attempts: 1, correct: 0, wrong: 1 }) + expect(scored.canonical.cost).toMatchObject({ total: 0.5, basis: 'observed', missing: 0 }) + expect(scored.heldOut.cost).toMatchObject({ total: null, basis: 'unknown', missing: 1 }) + expect(scored.notAttempted.length).toBeGreaterThan(0) + expect(scored.notAttempted).not.toContainEqual({ question: 'op.status-polls', variant: 0 }) + }) + + it('never turns an unreported cost into a zero', () => { + const scored = answers([ + { question: 'op.status-polls', answer: null, cost_usd: 2, cost_basis: 'estimated' }, + { question: 'op.runs', answer: null }, + ]) + expect(scored.canonical.cost).toMatchObject({ reported: 1, missing: 1, total: 2, basis: 'mixed' }) + expect(scored.canonical.falseNotInTrace).toBeGreaterThan(0) + }) + + it('renders one table row per attempted wording', () => { + const report = renderArmScore(answers([{ question: 'op.runs', answer: goldAnswer('op.runs') }])) + expect(report).toContain('# Audit benchmark score: test-arm') + expect(report).toContain('| op.runs |') + expect(report).toContain('Not attempted') + }) +}) + +describe('the runner', () => { + const run = (args: readonly string[], cwd: string) => + execFileAsync(process.execPath, ['--import', tsx, cli, ...args], { cwd, maxBuffer: 32 * 1024 * 1024 }) + + it('writes arm inputs without the gold, then scores an answers file', async () => { + const dir = await mkdtemp(join(tmpdir(), 'traces-bench-run-')) + try { + await run(['fixtures', '--out', join(dir, 'fixtures')], process.cwd()) + await run(['gold', '--out', join(dir, 'gold.json')], process.cwd()) + const manifest = JSON.parse(await readFile(join(dir, 'fixtures', 'manifest.json'), 'utf8')) as typeof bench.manifest + expect(manifest.files).toEqual(bench.manifest.files) + const prompts = (await readFile(join(dir, 'fixtures', 'prompts.jsonl'), 'utf8')).trim().split('\n') + expect(prompts.length).toBeGreaterThan(QUESTIONS.length) + await expect(readFile(join(dir, 'fixtures', 'gold.json'), 'utf8')).rejects.toThrow() + + const answersPath = join(dir, 'answers.json') + await writeFile(answersPath, JSON.stringify({ + arm: 'gold-replay', + answers: QUESTIONS.map((question) => ({ question: question.id, variant: 0, answer: bench.gold[question.id] })), + })) + await run(['score', answersPath, '--fixtures', join(dir, 'fixtures'), '--out', join(dir, 'report.md'), '--json', join(dir, 'score.json')], process.cwd()) + const parsed = JSON.parse(await readFile(join(dir, 'score.json'), 'utf8')) as { canonical: { correct: number; attempts: number } } + expect(parsed.canonical.correct).toBe(QUESTIONS.length) + expect(parsed.canonical.attempts).toBe(QUESTIONS.length) + expect(await readFile(join(dir, 'report.md'), 'utf8')).toContain('gold-replay') + + await writeFile(join(dir, 'fixtures', bench.manifest.files[0]!), 'tampered\n') + await expect(run(['score', answersPath, '--fixtures', join(dir, 'fixtures')], process.cwd())) + .rejects.toThrow(/differs from the generated bytes/) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/bench/audit/score.ts b/bench/audit/score.ts new file mode 100644 index 0000000..56332b9 --- /dev/null +++ b/bench/audit/score.ts @@ -0,0 +1,352 @@ +/** + * Exact-match scorer. No model judges any answer. + * + * Each schema leaf is scored on its own: counts, numbers, names and paths by + * equality, sets by set equality, times within 1 s, and quotes by verbatim + * text plus a citation that resolves to the gold record. A question is correct + * when every leaf is, wrong when none is, and partial otherwise. + */ + +import { type CitationIndex, normalizeText, type RecordRef } from './citations.js' +import type { GoldAnswers } from './fixtures.js' +import { type AnswerSchema, type Field, type Question, QUESTIONS, questionById } from './questions.js' + +export const TIME_TOLERANCE_MS = 1_000 + +export type Verdict = 'correct' | 'partial' | 'wrong' +export type CostBasis = 'observed' | 'estimated' + +/** One attempt by an arm at one question wording. */ +export interface AnswerRow { + question: string + /** 0 is the canonical wording; higher numbers are held-out paraphrases. */ + variant?: number + repeat?: number + /** The answer object, or null when the arm produced none. */ + answer: unknown + wall_ms?: number | null + model_calls?: number | null + tool_calls?: number | null + cost_usd?: number | null + cost_basis?: CostBasis | null + error?: string | null +} + +export interface AnswersFile { + arm: string + notes?: string + answers: AnswerRow[] +} + +export interface CitationCheck { + cite: string + /** The cite names a known record or span. */ + resolved: boolean + /** A named record holds the quoted text. */ + verified: boolean + /** The cite names the gold record. */ + onGold: boolean +} + +export interface LeafResult { + path: string + correct: boolean + /** The answer said "not in trace" (null) where the gold has a value. */ + falseNotInTrace: boolean + citations: CitationCheck[] +} + +export interface ScoredAnswer { + question: string + variant: number + repeat: number + verdict: Verdict + leaves: LeafResult[] +} + +const isNotInTrace = (value: unknown): boolean => + value === null || value === undefined || (typeof value === 'string' && /^not in trace$/i.test(value.trim())) + +function leaf(path: string, correct: boolean, answer: unknown, citations: CitationCheck[] = []): LeafResult { + return { path, correct, falseNotInTrace: !correct && isNotInTrace(answer), citations } +} + +function parseLineCite(cite: string): RecordRef { + const match = /^(.+):(\d+)$/.exec(cite) + if (!match) throw new Error(`gold cite is not :: ${cite}`) + return { file: match[1]!, line: Number(match[2]) } +} + +interface Quote { + text: string + cite: string +} + +const asQuote = (value: unknown): Quote | undefined => { + if (!value || typeof value !== 'object') return undefined + const { text, cite } = value as Record + return typeof text === 'string' && typeof cite === 'string' ? { text, cite } : undefined +} + +function checkQuote(answer: Quote, gold: Quote, index: CitationIndex): { correct: boolean; citation: CitationCheck } { + const target = parseLineCite(gold.cite) + const refs = index.resolve(answer.cite) + const verified = refs?.some((ref) => index.contains(ref, answer.text)) ?? false + const onGold = refs?.some((ref) => ref.file === target.file && ref.line === target.line) ?? false + const textMatches = normalizeText(answer.text) === normalizeText(gold.text) + return { + correct: textMatches && onGold, + citation: { cite: answer.cite, resolved: refs !== undefined, verified, onGold }, + } +} + +function sameSet(answer: unknown, gold: readonly unknown[], of: 'integer' | 'string'): boolean { + if (!Array.isArray(answer)) return false + const valid = of === 'integer' + ? answer.every((item) => Number.isSafeInteger(item)) + : answer.every((item) => typeof item === 'string') + if (!valid) return false + const normalize = (item: unknown): unknown => (typeof item === 'string' ? item.trim() : item) + const left = new Set(answer.map(normalize)) + const right = new Set(gold.map(normalize)) + return left.size === right.size && [...right].every((item) => left.has(item)) +} + +function scoreField(path: string, field: Field, answer: unknown, gold: unknown, index: CitationIndex): LeafResult[] { + switch (field.kind) { + case 'count': + return [leaf(path, Number.isSafeInteger(answer) && answer === gold, answer)] + case 'string': + return [leaf(path, typeof answer === 'string' && answer.trim() === gold, answer)] + case 'boolean': + return [leaf(path, answer === gold, answer)] + case 'time': { + const parsed = typeof answer === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(answer) ? Date.parse(answer) : Number.NaN + const correct = Number.isFinite(parsed) && Math.abs(parsed - Date.parse(String(gold))) <= TIME_TOLERANCE_MS + return [leaf(path, correct, answer)] + } + case 'set': + return [leaf(path, sameSet(answer, gold as unknown[], field.of), answer)] + case 'quote': { + const quote = asQuote(answer) + if (!quote) return [leaf(path, false, answer)] + const { correct, citation } = checkQuote(quote, gold as Quote, index) + return [leaf(path, correct, answer, [citation])] + } + case 'quotes': { + const golds = gold as Quote[] + const quotes = Array.isArray(answer) ? answer.map(asQuote) : [] + if (!Array.isArray(answer) || quotes.some((item) => item === undefined)) return [leaf(path, false, answer)] + const citations: CitationCheck[] = [] + const matched = new Set() + let unmatched = 0 + for (const quote of quotes as Quote[]) { + const checks = golds.map((target) => checkQuote(quote, target, index)) + const hit = checks.findIndex((check, position) => check.correct && !matched.has(position)) + const first = checks[0]?.citation ?? { cite: quote.cite, resolved: false, verified: false, onGold: false } + citations.push({ ...first, onGold: checks.some((check) => check.citation.onGold) }) + if (hit >= 0) matched.add(hit) + else unmatched += 1 + } + return [leaf(path, unmatched === 0 && matched.size === golds.length, answer, citations)] + } + case 'records': { + const golds = gold as Array> + const rows = Array.isArray(answer) + ? answer.filter((item): item is Record => Boolean(item) && typeof item === 'object') + : [] + const keyOf = (row: Record): unknown => (typeof row[field.key] === 'string' ? (row[field.key] as string).trim() : row[field.key]) + const answerKeys = rows.map(keyOf) + const keysCorrect = Array.isArray(answer) && rows.length === answer.length && sameSet(answerKeys, golds.map(keyOf), field.keyOf) + const results = [leaf(`${path}[*].${field.key}`, keysCorrect, answer)] + for (const target of golds) { + const key = keyOf(target) + const row = rows.find((candidate) => keyOf(candidate) === key) + for (const [name, subfield] of Object.entries(field.fields)) { + results.push(...scoreField(`${path}[${String(key)}].${name}`, subfield, row?.[name] ?? null, target[name], index)) + } + } + return results + } + } +} + +function scoreSchema(schema: AnswerSchema, answer: unknown, gold: Record, index: CitationIndex): LeafResult[] { + const object = answer && typeof answer === 'object' && !Array.isArray(answer) ? answer as Record : {} + return Object.entries(schema).flatMap(([name, field]) => scoreField(name, field, object[name] ?? null, gold[name], index)) +} + +export function scoreAnswer(question: Question, row: AnswerRow, gold: GoldAnswers, index: CitationIndex): ScoredAnswer { + const expected = gold[question.id] + if (!expected) throw new Error(`gold has no answer for ${question.id}`) + const leaves = scoreSchema(question.schema, row.answer, expected, index) + const correct = leaves.filter((item) => item.correct).length + return { + question: question.id, + variant: row.variant ?? 0, + repeat: row.repeat ?? 1, + verdict: correct === leaves.length ? 'correct' : correct === 0 ? 'wrong' : 'partial', + leaves, + } +} + +/** Reject a malformed answers file before scoring, with every problem listed. */ +export function parseAnswersFile(value: unknown): AnswersFile { + const problems: string[] = [] + const file = value as Partial | null + if (!file || typeof file !== 'object') throw new Error('answers file must be a JSON object') + if (typeof file.arm !== 'string' || file.arm.length === 0) problems.push('arm must be a non-empty string') + if (!Array.isArray(file.answers)) problems.push('answers must be an array') + for (const [position, row] of (Array.isArray(file.answers) ? file.answers : []).entries()) { + const question = typeof row?.question === 'string' ? questionById(row.question) : undefined + if (!question) problems.push(`answers[${position}].question is not a known question id`) + const variant = row?.variant ?? 0 + if (question && (!Number.isInteger(variant) || variant < 0 || variant > question.paraphrases.length)) { + problems.push(`answers[${position}].variant must be 0 to ${question.paraphrases.length}`) + } + if (row && !('answer' in row)) problems.push(`answers[${position}].answer is missing (use null for no answer)`) + if (row?.cost_basis != null && row.cost_basis !== 'observed' && row.cost_basis !== 'estimated') { + problems.push(`answers[${position}].cost_basis must be observed, estimated, or null`) + } + } + if (problems.length > 0) throw new Error(`invalid answers file:\n- ${problems.join('\n- ')}`) + return file as AnswersFile +} + +export interface Distribution { + /** Attempts that reported the measure. */ + reported: number + /** Attempts that did not; never counted as zero. */ + missing: number + median: number | null + max: number | null + total: number | null +} + +function distribution(values: ReadonlyArray): Distribution { + const reported = values.filter((value): value is number => typeof value === 'number' && Number.isFinite(value)).sort((a, b) => a - b) + const middle = Math.floor(reported.length / 2) + return { + reported: reported.length, + missing: values.length - reported.length, + median: reported.length === 0 ? null : reported.length % 2 === 1 ? reported[middle]! : (reported[middle - 1]! + reported[middle]!) / 2, + max: reported.at(-1) ?? null, + total: reported.length === 0 ? null : reported.reduce((sum, value) => sum + value, 0), + } +} + +export interface Tally { + attempts: number + correct: number + partial: number + wrong: number + /** Leaves answered "not in trace" where the gold has a value. */ + falseNotInTrace: number + citations: { total: number; resolved: number; verified: number; onGold: number } + wallMs: Distribution + modelCalls: Distribution + toolCalls: Distribution + cost: Distribution & { basis: CostBasis | 'mixed' | 'unknown' } +} + +export interface QuestionScore extends Tally { + question: string + variant: number + heldOut: boolean +} + +export interface ArmScore { + arm: string + notes?: string + questions: QuestionScore[] + canonical: Tally + heldOut: Tally + /** Question wordings with no attempt in the answers file. */ + notAttempted: Array<{ question: string; variant: number }> + attempts: ScoredAnswer[] +} + +function tally(rows: ReadonlyArray<{ row: AnswerRow; scored: ScoredAnswer }>): Tally { + const citations = rows.flatMap(({ scored }) => scored.leaves.flatMap((item) => item.citations)) + const bases = new Set(rows.map(({ row }) => (row.cost_usd == null ? null : row.cost_basis ?? null))) + const cost = distribution(rows.map(({ row }) => row.cost_usd)) + const knownBases = [...bases].filter((basis): basis is CostBasis => basis !== null) + return { + attempts: rows.length, + correct: rows.filter(({ scored }) => scored.verdict === 'correct').length, + partial: rows.filter(({ scored }) => scored.verdict === 'partial').length, + wrong: rows.filter(({ scored }) => scored.verdict === 'wrong').length, + falseNotInTrace: rows.reduce((sum, { scored }) => sum + scored.leaves.filter((item) => item.falseNotInTrace).length, 0), + citations: { + total: citations.length, + resolved: citations.filter((item) => item.resolved).length, + verified: citations.filter((item) => item.verified).length, + onGold: citations.filter((item) => item.onGold).length, + }, + wallMs: distribution(rows.map(({ row }) => row.wall_ms)), + modelCalls: distribution(rows.map(({ row }) => row.model_calls)), + toolCalls: distribution(rows.map(({ row }) => row.tool_calls)), + cost: { + ...cost, + basis: cost.reported === 0 ? 'unknown' : knownBases.length === 1 && !bases.has(null) ? knownBases[0]! : 'mixed', + }, + } +} + +export function scoreArm(file: AnswersFile, gold: GoldAnswers, index: CitationIndex): ArmScore { + const scored = file.answers.map((row) => ({ row, scored: scoreAnswer(questionById(row.question)!, row, gold, index) })) + const questions: QuestionScore[] = [] + const notAttempted: ArmScore['notAttempted'] = [] + for (const question of QUESTIONS) { + for (let variant = 0; variant <= question.paraphrases.length; variant += 1) { + const rows = scored.filter(({ scored: item }) => item.question === question.id && item.variant === variant) + if (rows.length === 0) notAttempted.push({ question: question.id, variant }) + else questions.push({ question: question.id, variant, heldOut: variant > 0, ...tally(rows) }) + } + } + return { + arm: file.arm, + ...(file.notes ? { notes: file.notes } : {}), + questions, + canonical: tally(scored.filter(({ scored: item }) => item.variant === 0)), + heldOut: tally(scored.filter(({ scored: item }) => item.variant > 0)), + notAttempted, + attempts: scored.map(({ scored: item }) => item), + } +} + +const fmt = (value: number | null, digits = 0): string => (value === null ? 'n/a' : value.toFixed(digits)) + +function costCell(cost: Tally['cost']): string { + if (cost.total === null) return 'unknown' + const missing = cost.missing > 0 ? `, ${cost.missing} unknown` : '' + return `$${cost.total.toFixed(4)} ${cost.basis}${missing}` +} + +function row(label: string, item: Tally): string { + const cites = item.citations.total === 0 + ? 'none' + : `${item.citations.verified}/${item.citations.total} verify, ${item.citations.onGold} on gold` + return `| ${label} | ${item.correct}/${item.partial}/${item.wrong} of ${item.attempts} | ${item.falseNotInTrace} | ${cites} | ${fmt(item.wallMs.median)} / ${fmt(item.wallMs.max)} | ${fmt(item.modelCalls.total)} | ${fmt(item.toolCalls.total)} | ${costCell(item.cost)} |` +} + +export function renderArmScore(score: ArmScore): string { + const header = [ + '| Question | Correct/partial/wrong | False "not in trace" | Citations | Wall ms median / max | Model calls | Tool calls | Cost |', + '|---|---|---|---|---|---|---|---|', + ] + return [ + `# Audit benchmark score: ${score.arm}`, + '', + ...(score.notes ? [score.notes, ''] : []), + ...header, + row('all canonical', score.canonical), + row('all held-out', score.heldOut), + ...score.questions.map((item) => row(`${item.question}${item.heldOut ? ` (paraphrase ${item.variant})` : ''}`, item)), + '', + score.notAttempted.length === 0 + ? 'Every question wording was attempted.' + : `Not attempted (${score.notAttempted.length}): ${score.notAttempted.map((item) => `${item.question}#${item.variant}`).join(', ')}.`, + '', + ].join('\n') +} diff --git a/package.json b/package.json index 97beb21..6be14e9 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ }, "scripts": { "dev": "tsx src/cli.ts", + "bench:audit": "tsx bench/audit/cli.ts", "build": "tsup src/index.ts src/cli.ts --format esm --target node22 --clean --dts", "check:package": "node scripts/check-package-bin.mjs", "check:source": "node scripts/check-source-text.mjs", diff --git a/tsconfig.json b/tsconfig.json index 3080bf5..4ad3f93 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -15,5 +15,5 @@ "@tangle-network/traces": ["./src/index.ts"] } }, - "include": ["src", "tests", "examples"] + "include": ["src", "tests", "examples", "bench"] } From 9cce570874c412458a675928e7dce9d15999b06c Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 10 Sep 2026 01:41:58 -0700 Subject: [PATCH 2/3] fix(bench): re-derive the run and role answer key from the fixture bytes The audit gold for `op.runs` and `op.role` was hand-kept bookkeeping that no test compared against the generated JSONL, so the guarantee the benchmark rests on, that the gold equals a re-derivation of the bytes, had a hole exactly where the counting was manual. Changing `runs.launched += 2` to `+= 3` in the generator left all 59 tests green while two of the nineteen questions would have marked every arm wrong with no signal. The key itself was already correct. Both answers now come from the output lines. Every `started run (spec [, variant ])` line, across direct commands and exec scripts alike, gives `launched`, `specs` and `beta_probe_variants`; nonzero-exit `labctl run` calls give `failed_launches`, and the two together must account for every direct `labctl run` call; the merge confirmations, the same launches and the `spawn_agent` calls give `op.role`. Each hand-kept field was mutated to confirm the new assertions fail on it. The rest are the review's non-blocking findings: - A gold leaf with no value is now scorable: `null`, and only `null`, answers it. Before, an optional field was compared against `Date.parse(String( undefined))` and no answer could be correct. - A span carrying no source record no longer resolves. 176 of 1,438 span keys mapped to an empty list, which reported in the same shape as a citation of the wrong record. - The two aggregate score rows count every wording the benchmark asks, so answering only the five easiest no longer reports 5/5. - `cost.basis` reads the reported costs alone; an omitted cost is already counted as missing. - `assertUnmodified` covers `manifest.json` and `prompts.jsonl` too, so an arm cannot reword its own prompts. Files the arm added stay ignored, and the comment says why that is safe. - The "records are at least 2 s apart" claim is narrowed to what is true and now asserted: every scored timestamp is further than the 1 s tolerance from any other record time in its file. The child's inherited history shares the fork timestamp, and nothing scored reads a time from it. - The README names the sixteen answers that are re-derived and the three that are asserted by construction. - The generated file order and the manifest assertion use one comparator. - `cli.ts` writes its usage to stderr when given no command, as every other failure path does. Co-Authored-By: Claude Opus 5 (1M context) --- bench/audit/README.md | 44 ++++++++++++----- bench/audit/citations.ts | 4 ++ bench/audit/cli.ts | 37 +++++++++----- bench/audit/fixtures.test.ts | 94 +++++++++++++++++++++++++++++++----- bench/audit/fixtures.ts | 12 +++-- bench/audit/score.test.ts | 52 ++++++++++++++++++-- bench/audit/score.ts | 33 +++++++++---- 7 files changed, 225 insertions(+), 51 deletions(-) diff --git a/bench/audit/README.md b/bench/audit/README.md index 2ebc9a6..cca49d8 100644 --- a/bench/audit/README.md +++ b/bench/audit/README.md @@ -29,9 +29,18 @@ The answer key therefore comes from the plan, never from parsing the files back the gold went through the adapter, the same defect would quietly rewrite the answer key and the benchmark would score nothing. -`fixtures.test.ts` closes the other half of that loop: it re-derives every planted fact by -reading the generated JSONL directly, with no adapter and no access to the generator's -bookkeeping, and fails if the plan and the bytes disagree. +`fixtures.test.ts` closes the other half of that loop: it reads the generated JSONL +directly, with no adapter and no access to the generator's bookkeeping, and fails when the +plan and the bytes disagree. + +Sixteen of the nineteen answers are re-derived that way, counted or read straight back out +of the bytes: `op.status-polls`, `op.subagents`, `op.pull-requests`, `op.runs`, `op.role`, +`op.last-human-turn`, `op.changed-files`, `op.exit-codes`, `op.tokens`, `op.time-bounds`, +`op.large-output`, `child.lineage`, `child.own-work`, `claude.tasks`, `claude.bash` and +`claude.first-bash-error`. Three are asserted by construction instead. `op.corrections` +rests on a judgement the generator makes, which human messages are corrections; the test +checks that each quoted correction is the verbatim text of the record it cites, not that +the set is complete. `op.local-copy` and `child.spawned` are literals. ## Sessions and what they plant @@ -83,20 +92,26 @@ exercises. Exact match. No model judges any answer. - Counts, numbers, names, paths, booleans: equality. -- Times: correct within 1 s. Neighboring records in the fixtures are at least 2 s apart, so - the tolerance can never accept an adjacent record's time. +- Times: correct within 1 s. Every timestamp the gold scores is more than 1 s from any + other record time in its file, so the tolerance can never accept a neighboring record's + time; `fixtures.test.ts` asserts that. The spacing is not uniform everywhere: the child's + inherited history all carries the fork timestamp, as Codex rewrites it, and no scored + time is read from that block. - Sets: set equality, order ignored, no missing and no extra member. - Quotes: the text verbatim (whitespace-insensitive) **and** a citation that resolves to the gold record. A citation may be `:`, a span id, or a `trace:///span/` URI; span ids resolve through the source-record offsets the adapters attach, so a span id counts only when the span really came from that record. +- A leaf the trace has no value for: `null`, and only `null`, is correct. - A question is `correct` when every leaf is correct, `wrong` when none is, `partial` otherwise. -Two things are counted rather than averaged away. A leaf answered `null` where the gold has -a value is reported separately as a false "not in trace"; a confident wrong answer and a -refusal are different failures. And an unreported cost stays `missing` — it never becomes -zero. +Three things are counted rather than averaged away. A leaf answered `null` where the gold +has a value is reported separately as a false "not in trace"; a confident wrong answer and +a refusal are different failures. An unreported cost stays `missing` — it never becomes +zero. And the two aggregate rows are scored over every wording the benchmark asks, not over +the attempts made, so an arm that answers only the easy questions reports its skips in the +same cell a reader compares. ## Running it @@ -112,8 +127,11 @@ pnpm bench:audit score /tmp/arm-answers.json --fixtures /tmp/audit-bench --out / ``` `--fixtures` is optional; without it the runner regenerates the tree in a temporary -directory. When it is given, every file in it must still match the generated bytes, so an -arm cannot be scored against a tree it edited. +directory. When it is given, every file the runner wrote into it — the sessions, +`manifest.json` and `prompts.jsonl` — must still match the generated bytes, so an arm +cannot be scored against a tree it edited, or against prompts it reworded. Files the arm +added are ignored: scoring reads the session paths from the manifest it regenerates in +memory, never from the directory listing. `prompts.jsonl` in the fixtures directory holds one row per question wording: the canonical question at `variant: 0` and each held-out paraphrase above it, each with the exact prompt @@ -151,8 +169,8 @@ is reported as a number. `cost_basis` says whether `cost_usd` was observed or es `pnpm test` runs the deterministic half: - `fixtures.test.ts` — the generator is byte-identical across runs, every planted fact is - present, and the gold equals a re-derivation of the fixtures that never touches the - generator's bookkeeping or an adapter. + present, and each re-derived answer above equals a re-derivation of the fixtures that + never touches the generator's bookkeeping or an adapter. - `score.test.ts` — the gold, submitted as an arm, scores every question correct through the same path a real arm takes; and the scorer's leaf rules, citation resolution, answers-file validation, tallies, and the runner end to end. diff --git a/bench/audit/citations.ts b/bench/audit/citations.ts index 152b87f..48dbc3a 100644 --- a/bench/audit/citations.ts +++ b/bench/audit/citations.ts @@ -146,6 +146,10 @@ export async function spanRecordMap(root: string, manifest: BenchManifest): Prom } } const records = [...refs.values()] + // A span that carries no source record names nothing. Mapping it to an empty list + // would report a citation of it as resolved, which is what citing a real but wrong + // record looks like, and the two failures have to stay apart in the report. + if (records.length === 0) continue spans.set(span.span_id, records) for (const key of ['traces.codex.source_span_id', 'traces.claude.source_span_id']) { const alias = span.attributes[key] diff --git a/bench/audit/cli.ts b/bench/audit/cli.ts index 0983e67..db6918b 100644 --- a/bench/audit/cli.ts +++ b/bench/audit/cli.ts @@ -65,33 +65,46 @@ async function writeOut(path: string, content: string): Promise { await writeFile(path, content) } +/** Every file the runner writes into an arm's directory, keyed by its relative path. */ +function armInputs(bench: GeneratedBench): Map { + const inputs = new Map(bench.files.map((file) => [file.path, file.content])) + inputs.set('manifest.json', `${JSON.stringify(bench.manifest, null, 2)}\n`) + inputs.set('prompts.jsonl', `${promptRows(bench.manifest).map((row) => JSON.stringify(row)).join('\n')}\n`) + return inputs +} + /** Write the fixture tree plus the manifest and prompts an arm needs. */ async function writeArmInputs(root: string, bench: GeneratedBench): Promise { - await writeFixtures(root, bench) - await writeOut(join(root, 'manifest.json'), `${JSON.stringify(bench.manifest, null, 2)}\n`) - const prompts = promptRows(bench.manifest).map((row) => JSON.stringify(row)).join('\n') - await writeOut(join(root, 'prompts.jsonl'), `${prompts}\n`) + for (const [path, content] of armInputs(bench)) await writeOut(join(root, path), content) } /** * A fixture directory is usable only when it still holds the generated bytes. * Scoring an arm against a tree the arm could have rewritten would grade the - * tree, not the arm. + * tree, not the arm. This covers every file the runner wrote, the sessions plus + * `manifest.json` and `prompts.jsonl`, so an arm cannot quietly reword its own + * prompts either. Files the arm added are left alone: scoring reads the session + * paths from the manifest it just generated, never from the directory listing, + * so a file the generator did not write is never read. */ async function assertUnmodified(root: string, bench: GeneratedBench): Promise { - for (const file of bench.files) { - const path = join(root, file.path) - const actual = await readFile(path, 'utf8').catch(() => undefined) - if (actual === undefined) throw new Error(`fixture directory is missing ${file.path}`) - if (actual !== file.content) throw new Error(`fixture file differs from the generated bytes: ${file.path}`) + for (const [file, content] of armInputs(bench)) { + const actual = await readFile(join(root, file), 'utf8').catch(() => undefined) + if (actual === undefined) throw new Error(`fixture directory is missing ${file}`) + if (actual !== content) throw new Error(`fixture file differs from the generated bytes: ${file}`) } } async function main(argv: readonly string[]): Promise { const [command, ...rest] = argv - if (!command || command === '--help' || command === '-h') { + if (command === '--help' || command === '-h') { console.log(USAGE) - return command ? 0 : 1 + return 0 + } + if (!command) { + // Every other failure path writes to stderr, so this one does too. + console.error(USAGE) + return 1 } const { positional, flags } = parseArgs(rest) const bench = generateBench() diff --git a/bench/audit/fixtures.test.ts b/bench/audit/fixtures.test.ts index 6bb16bd..3802b74 100644 --- a/bench/audit/fixtures.test.ts +++ b/bench/audit/fixtures.test.ts @@ -19,6 +19,7 @@ import { ClaudeAdapter } from '../../src/adapters/claude.js' import { CodexAdapter } from '../../src/adapters/codex.js' import { generateBench, writeFixtures } from './fixtures.js' import { QUESTIONS, answerJsonSchema, promptRows } from './questions.js' +import { TIME_TOLERANCE_MS } from './score.js' const bench = generateBench() const fileByPath = new Map(bench.files.map((file) => [file.path, file.content])) @@ -76,6 +77,30 @@ const operatorCalls = codexCalls(operator) const commandOf = (call: CodexCall): string => String((JSON.parse(call.argument) as { cmd?: string }).cmd ?? '') const execCalls = operatorCalls.filter((call) => call.name === 'exec_command') const scripts = operatorCalls.filter((call) => call.name === 'exec') +const spawnCalls = operatorCalls.filter((call) => call.name === 'spawn_agent') + +/** A run one launch reported as started, read from the output line that announced it. */ +interface StartedRun { + id: string + spec: string + variant?: string +} + +/** Every launch these outputs announced, whether the command ran directly or inside a script. */ +function startedRuns(calls: readonly CodexCall[]): StartedRun[] { + return calls.flatMap((call) => + [...call.output.matchAll(/^started run (\S+) \(spec ([^,)]+)(?:, variant ([^)]+))?\)$/gm)] + .map((match) => ({ id: match[1]!, spec: match[2]!, ...(match[3] ? { variant: match[3] } : {}) })), + ) +} + +/** Pull request numbers a merge confirmation names in a tool output. */ +const mergedIn = (text: string): number[] => + [...text.matchAll(/Squashed and merged pull request [^#]+#(\d+)/g)].map((match) => Number(match[1])) + +const directLaunches = execCalls.filter((call) => commandOf(call).startsWith('labctl run ')) +const failedLaunches = directLaunches.filter((call) => /Process exited with code (?!0)/.test(call.output)) +const launches = startedRuns([...execCalls, ...scripts]) /** Text of a user message, or undefined for any other record. */ function userText(row: CodexRow): string | undefined { @@ -90,6 +115,19 @@ const humanTurns = operator .filter((item): item is { row: CodexRow; text: string } => item.text !== undefined) .filter((item) => !item.text.startsWith('<') && !item.text.startsWith('#')) +/** Every gold value the scorer compares as a time, wherever it sits in an answer. */ +const scoredTimes = QUESTIONS.flatMap((question) => { + const gold = bench.gold[question.id]! + return Object.entries(question.schema).flatMap(([name, field]) => { + if (field.kind === 'time') return [String(gold[name])] + if (field.kind !== 'records') return [] + const rows = gold[name] as Array> + return Object.entries(field.fields) + .filter(([, subfield]) => subfield.kind === 'time') + .flatMap(([subfield]) => rows.map((row) => row[subfield]).filter((value) => value != null).map(String)) + }) +}) + describe('audit benchmark generator', () => { it('produces the same bytes on every run', () => { const again = generateBench() @@ -120,6 +158,19 @@ describe('audit benchmark generator', () => { } }) + it('keeps every scored time further apart than the scorer tolerates', () => { + // The 1 s tolerance is only safe while no other record in the same file carries a time + // within 1 s of a scored one; otherwise a neighbor's time would be accepted as correct. + const codexTimes = [operatorPath, childPath].map((path) => codexRows(path).map((row) => Date.parse(row.timestamp))) + expect(scoredTimes.length).toBeGreaterThan(5) + for (const value of scoredTimes) { + const at = Date.parse(value) + const times = codexTimes.find((list) => list.includes(at)) + expect(times, `${value} is not the timestamp of any record`).toBeDefined() + expect(times!.filter((other) => other !== at && Math.abs(other - at) <= TIME_TOLERANCE_MS)).toEqual([]) + } + }) + it('gives every question wording a prompt naming its session', () => { const rows = promptRows(bench.manifest) expect(rows).toHaveLength(QUESTIONS.reduce((sum, question) => sum + 1 + question.paraphrases.length, 0)) @@ -155,10 +206,9 @@ describe('planted facts in the operator session', () => { }) it('spawns sixteen agents, one of which fails', () => { - const spawns = operatorCalls.filter((call) => call.name === 'spawn_agent') - expect(spawns).toHaveLength(16) - expect(spawns.filter((call) => call.output.startsWith('spawn_agent failed'))).toHaveLength(1) - const succeeded = spawns.filter((call) => !call.output.startsWith('spawn_agent failed')) + expect(spawnCalls).toHaveLength(16) + expect(spawnCalls.filter((call) => call.output.startsWith('spawn_agent failed'))).toHaveLength(1) + const succeeded = spawnCalls.filter((call) => !call.output.startsWith('spawn_agent failed')) const names = [...new Set(succeeded.map((call) => String((JSON.parse(call.argument) as { task_name: string }).task_name)))].sort() expect(bench.gold['op.subagents']).toEqual({ spawn_calls: 16, failed_spawns: 1, task_names: names }) }) @@ -177,10 +227,9 @@ describe('planted facts in the operator session', () => { }) it('merges three pull requests three different ways', () => { - const merged = (text: string): number[] => [...text.matchAll(/Squashed and merged pull request [^#]+#(\d+)/g)].map((match) => Number(match[1])) - expect(execCalls.flatMap((call) => merged(call.output))).toEqual([41]) - expect(operatorCalls.filter((call) => call.name === 'write_stdin').flatMap((call) => merged(call.output))).toEqual([42]) - expect(scripts.flatMap((call) => merged(call.output))).toEqual([43]) + expect(execCalls.flatMap((call) => mergedIn(call.output))).toEqual([41]) + expect(operatorCalls.filter((call) => call.name === 'write_stdin').flatMap((call) => mergedIn(call.output))).toEqual([42]) + expect(scripts.flatMap((call) => mergedIn(call.output))).toEqual([43]) const prs = bench.gold['op.pull-requests']!.prs as Array<{ number: number; merged_at?: string }> expect(prs.filter((pr) => pr.merged_at)).toHaveLength(3) // A merge that the harness refused must not count as the merge. @@ -188,13 +237,36 @@ describe('planted facts in the operator session', () => { }) it('repeats and cancels run commands', () => { - const launches = execCalls.filter((call) => commandOf(call).startsWith('labctl run ')) - const specs = launches.map((call) => commandOf(call).split(' ')[2]!) + // A direct launch either announced a run or exited nonzero; nothing else is a launch, + // so the announcements and the failures together account for every `labctl run` call. + expect(startedRuns(execCalls).length + failedLaunches.length).toBe(directLaunches.length) + // Two more launches happen inside an exec script, where no `labctl run` call record exists. + expect(startedRuns(scripts).length).toBeGreaterThan(0) + const specs = launches.map((run) => run.spec) expect(new Set(specs).size).toBeLessThan(specs.length) const cancels = execCalls.filter((call) => commandOf(call).startsWith('labctl cancel ')) const cancelled = cancels.filter((call) => call.output.includes('Process exited with code 0')) - expect(bench.gold['op.runs']!.cancelled).toBe(cancelled.length) expect(cancels.length).toBeGreaterThan(cancelled.length) + expect(bench.gold['op.runs']).toEqual({ + launched: launches.length, + failed_launches: failedLaunches.length, + cancelled: cancelled.length, + specs: [...new Set(specs)].sort(), + beta_probe_variants: [...new Set(launches.filter((run) => run.spec === 'beta-probe').map((run) => run.variant!))].sort(), + }) + }) + + it('acted as an operator, in the merges, launches and spawns the bytes show', () => { + // What makes this session the operator rather than an observer: it carries typed human + // turns, no parent thread claims it, and it merged, launched and spawned work itself. + expect(operator[0]!.payload.parent_thread_id).toBeUndefined() + expect(humanTurns.length).toBeGreaterThan(3) + expect(bench.gold['op.role']).toEqual({ + role: 'operator', + merged_prs: operatorCalls.flatMap((call) => mergedIn(call.output)).length, + launched_runs: launches.length, + spawn_calls: spawnCalls.length, + }) }) it('ends with a short human turn after a substantive one, then injected text', () => { diff --git a/bench/audit/fixtures.ts b/bench/audit/fixtures.ts index 922ee89..070a5e5 100644 --- a/bench/audit/fixtures.ts +++ b/bench/audit/fixtures.ts @@ -6,8 +6,12 @@ * the files back. The gold therefore does not depend on any traces adapter, and * an adapter defect shows up as a wrong answer instead of a wrong answer key. * - * Neighboring records are at least 2 s apart, so the scorer's 1 s tolerance for - * times can never accept the time of an adjacent record. + * Every timestamp the gold scores as a time is more than the scorer's 1 s + * tolerance away from any other record time in its file, so the tolerance can + * never accept a neighboring record's time. That is the property the tolerance + * rests on, and `fixtures.test.ts` asserts it. It is narrower than a uniform + * spacing: the child's inherited history all carries the fork timestamp, as + * Codex rewrites it, and no scored time is read from that block. */ import { mkdir, writeFile } from 'node:fs/promises' @@ -832,7 +836,9 @@ export function generateBench(): GeneratedBench { { path: operator.file, content: `${operator.rows.join('\n')}\n` }, { path: child.file, content: child.content }, ...claude.files, - ].sort((a, b) => a.path.localeCompare(b.path)) + // Code-unit order, the order `manifest.files` is asserted in and the order a + // plain `sort()` gives, rather than a locale-dependent collation. + ].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) const basenames = new Set(files.map((item) => item.path.split('/').at(-1))) if (basenames.size !== files.length) throw new Error('fixture file basenames must be unique so line citations resolve') return { diff --git a/bench/audit/score.test.ts b/bench/audit/score.test.ts index 383cdd6..ab6728f 100644 --- a/bench/audit/score.test.ts +++ b/bench/audit/score.test.ts @@ -94,6 +94,26 @@ describe('exact-match leaves', () => { expect(score('op.local-copy', { path: 'tools/mini-graph.mjs' }).leaves[0]!.falseNotInTrace).toBe(false) }) + it('accepts null for a leaf the trace has no value for', () => { + // No gold leaf is absent today, so this scores against a gold whose optional merge time + // is missing: an arm that reports null there is right, and one that invents a time is not. + const gold = structuredClone(bench.gold) + const prs = gold['op.pull-requests']!.prs as Array> + const merged = prs[0]!.merged_at + delete prs[0]!.merged_at + const answer = goldAnswer('op.pull-requests') + const rows = answer.prs as Array> + rows[0]!.merged_at = null + const scoreAgainst = (value: unknown) => { + rows[0]!.merged_at = value + return scoreAnswer(questionById('op.pull-requests')!, { question: 'op.pull-requests', answer }, gold, index) + } + const absent = scoreAgainst(null).leaves.find((leaf) => leaf.path === 'prs[41].merged_at')! + expect(absent).toMatchObject({ correct: true, falseNotInTrace: false }) + expect(scoreAgainst(null).verdict).toBe('correct') + expect(scoreAgainst(merged).leaves.find((leaf) => leaf.path === 'prs[41].merged_at')!.correct).toBe(false) + }) + it('scores an empty list as the right answer only where the gold is empty', () => { expect(score('child.spawned', { spawned_session_ids: [] }).verdict).toBe('correct') expect(score('child.spawned', { spawned_session_ids: ['made-up'] }).verdict).toBe('wrong') @@ -134,6 +154,13 @@ describe('quotes and citations', () => { expect(scored.leaves[0]!.citations[0]).toMatchObject({ resolved: true, verified: false, onGold: false }) }) + it('resolves a span id only when the span names a record', async () => { + const spans = await spanRecordMap(root, bench.manifest) + // An empty mapping would resolve, so a cite of a record-less span would be reported the + // same way as a cite of the wrong record. + expect([...spans].filter(([, refs]) => refs.length === 0)).toEqual([]) + }) + it('reports a citation that names no known record', () => { const scored = score('op.large-output', { last_line: { text: goldQuote.text, cite: 'nowhere.jsonl:12' } }) expect(scored.leaves[0]!.citations[0]).toMatchObject({ resolved: false, verified: false, onGold: false }) @@ -188,20 +215,30 @@ describe('arm scores', () => { expect(scored.notAttempted).not.toContainEqual({ question: 'op.status-polls', variant: 0 }) }) - it('never turns an unreported cost into a zero', () => { + it('never turns an unreported cost into a zero, and reads the basis off the reported ones', () => { const scored = answers([ { question: 'op.status-polls', answer: null, cost_usd: 2, cost_basis: 'estimated' }, { question: 'op.runs', answer: null }, ]) - expect(scored.canonical.cost).toMatchObject({ reported: 1, missing: 1, total: 2, basis: 'mixed' }) + expect(scored.canonical.cost).toMatchObject({ reported: 1, missing: 1, total: 2, basis: 'estimated' }) expect(scored.canonical.falseNotInTrace).toBeGreaterThan(0) + const twoBases = answers([ + { question: 'op.status-polls', answer: null, cost_usd: 2, cost_basis: 'estimated' }, + { question: 'op.runs', answer: null, cost_usd: 1, cost_basis: 'observed' }, + ]) + expect(twoBases.canonical.cost).toMatchObject({ reported: 2, missing: 0, total: 3, basis: 'mixed' }) + const noBasis = answers([{ question: 'op.runs', answer: null, cost_usd: 1 }]) + expect(noBasis.canonical.cost).toMatchObject({ reported: 1, total: 1, basis: 'unknown' }) }) - it('renders one table row per attempted wording', () => { + it('renders one table row per attempted wording, over the full wording count', () => { const report = renderArmScore(answers([{ question: 'op.runs', answer: goldAnswer('op.runs') }])) expect(report).toContain('# Audit benchmark score: test-arm') expect(report).toContain('| op.runs |') expect(report).toContain('Not attempted') + // Answering one easy question must not report 1/0/0 of 1: the denominator is every + // canonical wording, so skipping the rest is visible in the row a reader compares. + expect(report).toContain(`| all canonical | 1/0/0 of ${QUESTIONS.length} (${QUESTIONS.length - 1} not attempted) |`) }) }) @@ -231,6 +268,11 @@ describe('the runner', () => { expect(parsed.canonical.attempts).toBe(QUESTIONS.length) expect(await readFile(join(dir, 'report.md'), 'utf8')).toContain('gold-replay') + // The prompts an arm was given are checked too, not only the session bytes. + await writeFile(join(dir, 'fixtures', 'prompts.jsonl'), 'reworded\n') + await expect(run(['score', answersPath, '--fixtures', join(dir, 'fixtures')], process.cwd())) + .rejects.toThrow(/prompts\.jsonl/) + await writeFile(join(dir, 'fixtures', bench.manifest.files[0]!), 'tampered\n') await expect(run(['score', answersPath, '--fixtures', join(dir, 'fixtures')], process.cwd())) .rejects.toThrow(/differs from the generated bytes/) @@ -238,4 +280,8 @@ describe('the runner', () => { await rm(dir, { recursive: true, force: true }) } }) + + it('writes its usage to stderr when it is given no command', async () => { + await expect(run([], process.cwd())).rejects.toMatchObject({ code: 1, stdout: '', stderr: expect.stringContaining('Usage:') }) + }) }) diff --git a/bench/audit/score.ts b/bench/audit/score.ts index 56332b9..d4b27e9 100644 --- a/bench/audit/score.ts +++ b/bench/audit/score.ts @@ -3,8 +3,9 @@ * * Each schema leaf is scored on its own: counts, numbers, names and paths by * equality, sets by set equality, times within 1 s, and quotes by verbatim - * text plus a citation that resolves to the gold record. A question is correct - * when every leaf is, wrong when none is, and partial otherwise. + * text plus a citation that resolves to the gold record. A leaf the gold has no + * value for is correct only when the answer says "not in trace". A question is + * correct when every leaf is, wrong when none is, and partial otherwise. */ import { type CitationIndex, normalizeText, type RecordRef } from './citations.js' @@ -113,6 +114,9 @@ function sameSet(answer: unknown, gold: readonly unknown[], of: 'integer' | 'str } function scoreField(path: string, field: Field, answer: unknown, gold: unknown, index: CitationIndex): LeafResult[] { + // A gold leaf the trace has no value for is answered by "not in trace" and by nothing + // else, whatever its kind. Without this, an optional field could not be scored at all. + if (gold === null || gold === undefined) return [leaf(path, isNotInTrace(answer), answer)] switch (field.kind) { case 'count': return [leaf(path, Number.isSafeInteger(answer) && answer === gold, answer)] @@ -268,7 +272,9 @@ export interface ArmScore { function tally(rows: ReadonlyArray<{ row: AnswerRow; scored: ScoredAnswer }>): Tally { const citations = rows.flatMap(({ scored }) => scored.leaves.flatMap((item) => item.citations)) - const bases = new Set(rows.map(({ row }) => (row.cost_usd == null ? null : row.cost_basis ?? null))) + // An attempt that reported no cost says nothing about the basis of the ones that did; + // `missing` already carries the omission. + const bases = new Set(rows.filter(({ row }) => row.cost_usd != null).map(({ row }) => row.cost_basis ?? null)) const cost = distribution(rows.map(({ row }) => row.cost_usd)) const knownBases = [...bases].filter((basis): basis is CostBasis => basis !== null) return { @@ -288,7 +294,7 @@ function tally(rows: ReadonlyArray<{ row: AnswerRow; scored: ScoredAnswer }>): T toolCalls: distribution(rows.map(({ row }) => row.tool_calls)), cost: { ...cost, - basis: cost.reported === 0 ? 'unknown' : knownBases.length === 1 && !bases.has(null) ? knownBases[0]! : 'mixed', + basis: knownBases.length === 0 ? 'unknown' : knownBases.length === 1 && !bases.has(null) ? knownBases[0]! : 'mixed', }, } } @@ -323,25 +329,34 @@ function costCell(cost: Tally['cost']): string { return `$${cost.total.toFixed(4)} ${cost.basis}${missing}` } -function row(label: string, item: Tally): string { +/** + * One table row. `wordings` is the denominator a reader compares arms on: for the two + * aggregate rows it is every wording the benchmark asks, so an arm that skipped the hard + * questions is not rewarded with a smaller denominator. + */ +function row(label: string, item: Tally, wordings = item.attempts): string { const cites = item.citations.total === 0 ? 'none' : `${item.citations.verified}/${item.citations.total} verify, ${item.citations.onGold} on gold` - return `| ${label} | ${item.correct}/${item.partial}/${item.wrong} of ${item.attempts} | ${item.falseNotInTrace} | ${cites} | ${fmt(item.wallMs.median)} / ${fmt(item.wallMs.max)} | ${fmt(item.modelCalls.total)} | ${fmt(item.toolCalls.total)} | ${costCell(item.cost)} |` + const skipped = wordings - item.attempts + const verdicts = `${item.correct}/${item.partial}/${item.wrong} of ${wordings}${skipped > 0 ? ` (${skipped} not attempted)` : ''}` + return `| ${label} | ${verdicts} | ${item.falseNotInTrace} | ${cites} | ${fmt(item.wallMs.median)} / ${fmt(item.wallMs.max)} | ${fmt(item.modelCalls.total)} | ${fmt(item.toolCalls.total)} | ${costCell(item.cost)} |` } export function renderArmScore(score: ArmScore): string { const header = [ - '| Question | Correct/partial/wrong | False "not in trace" | Citations | Wall ms median / max | Model calls | Tool calls | Cost |', + '| Question | Correct/partial/wrong of wordings | False "not in trace" | Citations | Wall ms median / max | Model calls | Tool calls | Cost |', '|---|---|---|---|---|---|---|---|', ] + const canonicalWordings = QUESTIONS.length + const heldOutWordings = QUESTIONS.reduce((sum, question) => sum + question.paraphrases.length, 0) return [ `# Audit benchmark score: ${score.arm}`, '', ...(score.notes ? [score.notes, ''] : []), ...header, - row('all canonical', score.canonical), - row('all held-out', score.heldOut), + row('all canonical', score.canonical, canonicalWordings), + row('all held-out', score.heldOut, heldOutWordings), ...score.questions.map((item) => row(`${item.question}${item.heldOut ? ` (paraphrase ${item.variant})` : ''}`, item)), '', score.notAttempted.length === 0 From 8e3822f6e666cbf04fa7e2bf15cd78da5fae8f4b Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 10 Sep 2026 02:01:58 -0700 Subject: [PATCH 3/3] fix(bench): re-derive every pull request leaf from the fixture bytes `op.pull-requests` re-derived only the three PR numbers. Its nine other leaves - created_at, merged_at and reviewed_before_merge for 41/42/43 - were hand-kept bookkeeping no test compared against the records, so a wrong merge time or review flag would mark every arm wrong on the question that carries F3 and F4 with no signal anywhere. `fixtures.test.ts` now pairs each call with the outputs that answered it, including the outputs of later polls of a process the call backgrounded, and reads all ten leaves back out: the create call's own timestamp, the timestamp of the call that ran the merge the bytes confirm (excluding the merge the harness refused), and whether the last review listing shown before that call named a review. `op.pull-requests` is asserted whole. Mutating each of the six hand-set values, one at a time, now fails. Also from the review, none of it changing a gold value: - `child.spawned` is re-derived from the child's own spawn outputs, with the inherited spawn call asserted present, so the distractor the question exists for is pinned. Only `op.local-copy` and `op.corrections` remain asserted by construction. - The time-spacing test says which property it does not give. - `scoreArm` validates its input, so an unvalidated answers file reports every problem instead of throwing a bare TypeError. - The question names the empty-review-list case, so a leaf turns on retrieval rather than on a reading of the wording. - The bench README says the per-question rows use their own attempts as the denominator; the root README links `bench/audit/`. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 19 ++++++ bench/audit/README.md | 25 +++++--- bench/audit/fixtures.test.ts | 116 +++++++++++++++++++++++++++++++++-- bench/audit/questions.ts | 4 +- bench/audit/score.test.ts | 5 ++ bench/audit/score.ts | 7 ++- 6 files changed, 160 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ff5d32c..8a6b603 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Emitting the contract is the supported way to integrate a new system. The adapte - [Agent skills](#agent-skills) - [Library (SDK)](#library-sdk) - [Examples](#examples) +- [Audit benchmark](#audit-benchmark) - [Develop](#develop) ## Install @@ -756,6 +757,24 @@ Runnable, in [`examples/`](./examples): | [`register-harness.ts`](./examples/register-harness.ts) | add a new harness by implementing `HarnessTraceAdapter` | | [`external-engines.ts`](./examples/external-engines.ts) | drive HALO, Hodoscope, and an external PII scrubber | +## Audit benchmark + +[`bench/audit/`](./bench/audit) asks nineteen questions about three synthetic agent +sessions and scores an arm's answers by exact match, with no model judging anything. It +measures whether reading a session with `traces` beats reading the raw files, on facts a +single read cannot cover: counts over hundreds of records, timestamps of specific records, +and facts linked across tool calls and sessions. + +```bash +pnpm bench:audit fixtures --out /tmp/audit-bench # the tree an arm may read +pnpm bench:audit gold --out /tmp/audit-gold.json # the answer key, kept away from the arm +pnpm bench:audit score /tmp/arm-answers.json --fixtures /tmp/audit-bench +``` + +[`bench/audit/README.md`](./bench/audit/README.md) documents the questions, the scoring +rules, the answers-file format, and which answers are re-derived from the generated bytes. +The benchmark ships outside the published package; the model arms are run by hand. + ## Develop ```bash diff --git a/bench/audit/README.md b/bench/audit/README.md index cca49d8..291986e 100644 --- a/bench/audit/README.md +++ b/bench/audit/README.md @@ -33,14 +33,19 @@ the benchmark would score nothing. directly, with no adapter and no access to the generator's bookkeeping, and fails when the plan and the bytes disagree. -Sixteen of the nineteen answers are re-derived that way, counted or read straight back out -of the bytes: `op.status-polls`, `op.subagents`, `op.pull-requests`, `op.runs`, `op.role`, -`op.last-human-turn`, `op.changed-files`, `op.exit-codes`, `op.tokens`, `op.time-bounds`, -`op.large-output`, `child.lineage`, `child.own-work`, `claude.tasks`, `claude.bash` and -`claude.first-bash-error`. Three are asserted by construction instead. `op.corrections` -rests on a judgement the generator makes, which human messages are corrections; the test -checks that each quoted correction is the verbatim text of the record it cites, not that -the set is complete. `op.local-copy` and `child.spawned` are literals. +Seventeen of the nineteen answers are re-derived that way, counted or read straight back +out of the bytes: `op.status-polls`, `op.subagents`, `op.pull-requests`, `op.runs`, +`op.role`, `op.last-human-turn`, `op.changed-files`, `op.exit-codes`, `op.tokens`, +`op.time-bounds`, `op.large-output`, `child.lineage`, `child.own-work`, `child.spawned`, +`claude.tasks`, `claude.bash` and `claude.first-bash-error`. Every leaf is covered, not +just the keys: for `op.pull-requests` that means each create time, each merge time and each +`reviewed_before_merge`, re-derived by pairing calls with the outputs that answered them, +including the two links that exist only through a later poll of a backgrounded process. + +Two are asserted by construction instead. `op.corrections` rests on a judgement the +generator makes, which human messages are corrections; the test checks that each quoted +correction is the verbatim text of the record it cites, not that the set is complete. +`op.local-copy` is a literal. ## Sessions and what they plant @@ -111,7 +116,9 @@ has a value is reported separately as a false "not in trace"; a confident wrong a refusal are different failures. An unreported cost stays `missing` — it never becomes zero. And the two aggregate rows are scored over every wording the benchmark asks, not over the attempts made, so an arm that answers only the easy questions reports its skips in the -same cell a reader compares. +same cell a reader compares. The per-question rows under them keep their own attempts as +the denominator, so three repeats of one wording read `3/0/0 of 3`; compare arms on the +two aggregate rows, not on those. ## Running it diff --git a/bench/audit/fixtures.test.ts b/bench/audit/fixtures.test.ts index 3802b74..75354e3 100644 --- a/bench/audit/fixtures.test.ts +++ b/bench/audit/fixtures.test.ts @@ -98,6 +98,93 @@ function startedRuns(calls: readonly CodexCall[]): StartedRun[] { const mergedIn = (text: string): number[] => [...text.matchAll(/Squashed and merged pull request [^#]+#(\d+)/g)].map((match) => Number(match[1])) +/** The shell command a call ran, or the source of the script it ran; empty for any other call. */ +const sourceOf = (call: CodexCall): string => + call.name === 'exec_command' ? commandOf(call) : call.name === 'exec' ? call.argument : '' + +/** The process a call left running, when its output reports a session id instead of an exit. */ +function backgroundedSession(call: CodexCall): number | undefined { + const match = /^Process running with session ID (\d+)$/m.exec(call.output) + return match ? Number(match[1]) : undefined +} + +/** The background session a `write_stdin` call polled, or undefined for any other call. */ +function polledSession(call: CodexCall): number | undefined { + if (call.name !== 'write_stdin') return undefined + return Number((JSON.parse(call.argument) as { session_id: number }).session_id) +} + +/** + * Everything a call ended up reporting: its own output, plus the output of every later poll + * of the process it backgrounded. A `gh pr create` whose URL arrives only in a poll, and a + * `gh pr merge` whose confirmation does the same, are both readable only this way. + */ +function reported(calls: readonly CodexCall[], call: CodexCall): string { + const session = backgroundedSession(call) + if (session === undefined) return call.output + const polls = calls.filter((other) => other.call.line > call.call.line && polledSession(other) === session) + return [call.output, ...polls.map((poll) => poll.output)].join('\n') +} + +/** The command's own output, below the header the shell tool prefixes to it. */ +function shellBody(output: string): string { + const marker = '\nOutput:\n' + const index = output.indexOf(marker) + if (index < 0) throw new Error('a shell tool output carried no body') + return output.slice(index + marker.length) +} + +/** One pull request as the records describe it, in the shape the answer takes. */ +interface DerivedPullRequest { + number: number + created_at: string + merged_at?: string + reviewed_before_merge?: boolean +} + +/** + * Every pull request leaf, read back out of the records with no generator bookkeeping. + * + * `created_at` and `merged_at` are the timestamps of the call records themselves, as the + * question asks, not of the outputs that answered them. A merge counts only when the bytes + * confirm it, so the merge the harness refused is not one. `reviewed_before_merge` is + * whether the last review listing the session saw for that pull request before the merge + * call named any review. + */ +function pullRequests(calls: readonly CodexCall[]): DerivedPullRequest[] { + const created = new Map() + for (const call of calls) { + if (!sourceOf(call).includes('gh pr create')) continue + const numbers = [...reported(calls, call).matchAll(/\/pull\/(\d+)/g)].map((match) => Number(match[1])) + if (numbers.length !== 1) throw new Error(`a gh pr create call reported ${numbers.length} pull request URLs`) + created.set(numbers[0]!, call) + } + const merged = new Map() + for (const call of calls) { + for (const match of sourceOf(call).matchAll(/gh pr merge (\d+)/g)) { + const number = Number(match[1]) + if (!mergedIn(reported(calls, call)).includes(number)) continue + if (merged.has(number)) throw new Error(`two calls claim the merge of pull request ${number}`) + merged.set(number, call) + } + } + return [...created.entries()].sort(([a], [b]) => a - b).map(([number, create]) => { + const merge = merged.get(number) + if (!merge) return { number, created_at: create.call.timestamp } + const views = calls.filter((call) => + call.call.line < merge.call.line && sourceOf(call).includes(`gh pr view ${number} --json reviews`)) + const last = views.at(-1) + if (!last) throw new Error(`nothing showed a review of pull request ${number} before its merge`) + const { reviews } = JSON.parse(shellBody(last.output)) as { reviews: unknown[] } + return { + number, + created_at: create.call.timestamp, + merged_at: merge.call.timestamp, + reviewed_before_merge: reviews.length > 0, + } + }) +} + const directLaunches = execCalls.filter((call) => commandOf(call).startsWith('labctl run ')) const failedLaunches = directLaunches.filter((call) => /Process exited with code (?!0)/.test(call.output)) const launches = startedRuns([...execCalls, ...scripts]) @@ -161,6 +248,8 @@ describe('audit benchmark generator', () => { it('keeps every scored time further apart than the scorer tolerates', () => { // The 1 s tolerance is only safe while no other record in the same file carries a time // within 1 s of a scored one; otherwise a neighbor's time would be accepted as correct. + // This says nothing about whether a scored time belongs to the right record: it proves + // the tolerance is safe, not the answer. Each question's own test proves the record. const codexTimes = [operatorPath, childPath].map((path) => codexRows(path).map((row) => Date.parse(row.timestamp))) expect(scoredTimes.length).toBeGreaterThan(5) for (const value of scoredTimes) { @@ -223,19 +312,28 @@ describe('planted facts in the operator session', () => { const create43 = execCalls.find((call) => commandOf(call).includes('docs(retry): budget guide')) expect(create43).toBeDefined() expect(numbersIn(create43!.output)).toEqual([]) - expect((bench.gold['op.pull-requests']!.prs as Array<{ number: number }>).map((pr) => pr.number)).toEqual([41, 42, 43]) }) it('merges three pull requests three different ways', () => { expect(execCalls.flatMap((call) => mergedIn(call.output))).toEqual([41]) expect(operatorCalls.filter((call) => call.name === 'write_stdin').flatMap((call) => mergedIn(call.output))).toEqual([42]) expect(scripts.flatMap((call) => mergedIn(call.output))).toEqual([43]) - const prs = bench.gold['op.pull-requests']!.prs as Array<{ number: number; merged_at?: string }> - expect(prs.filter((pr) => pr.merged_at)).toHaveLength(3) // A merge that the harness refused must not count as the merge. expect(execCalls.some((call) => commandOf(call) === 'gh pr merge 41 --squash' && call.output.includes('not mergeable'))).toBe(true) }) + it('re-derives every pull request answer from the records', () => { + // Every leaf comes back out of the bytes: the create call's own timestamp, the timestamp + // of the call that ran the merge the records confirm, and the review listing the session + // last saw before that call. Two of those links exist only through a later poll of a + // backgrounded process, which is the F4 difficulty this question carries. + const derived = pullRequests(operatorCalls) + expect(derived.map((pr) => pr.number)).toEqual([41, 42, 43]) + expect(derived.filter((pr) => pr.merged_at)).toHaveLength(3) + expect(new Set(derived.map((pr) => pr.reviewed_before_merge))).toEqual(new Set([true, false])) + expect(bench.gold['op.pull-requests']).toEqual({ prs: derived }) + }) + it('repeats and cancels run commands', () => { // A direct launch either announced a run or exited nonzero; nothing else is a launch, // so the announcements and the failures together account for every `labctl run` call. @@ -355,7 +453,17 @@ describe('planted facts in the forked child session', () => { own_tool_calls: calls.length, failed_commands: failed.length, }) - expect(bench.gold['child.spawned']).toEqual({ spawned_session_ids: [] }) + // The child carries exactly one spawn_agent call, inherited at the fork timestamp, and + // made none of its own. Counting the inherited call is the mistake this question tests + // for, so the derivation reads the session ids off the child's own spawn outputs. + const spawnsIn = (rows: readonly CodexRow[]): CodexRow[] => + rows.filter((row) => row.payload.type === 'function_call' && row.payload.name === 'spawn_agent') + expect(spawnsIn(child.slice(1).filter((row) => row.timestamp === child[0]!.timestamp))).toHaveLength(1) + expect(spawnsIn(own)).toHaveLength(0) + const spawned = codexCalls(own) + .filter((call) => call.name === 'spawn_agent') + .map((call) => String((JSON.parse(call.output) as { agent_id: string }).agent_id)) + expect(bench.gold['child.spawned']).toEqual({ spawned_session_ids: spawned }) }) }) diff --git a/bench/audit/questions.ts b/bench/audit/questions.ts index 6bfda32..5d8ddb8 100644 --- a/bench/audit/questions.ts +++ b/bench/audit/questions.ts @@ -56,8 +56,8 @@ export const QUESTIONS: readonly Question[] = [ id: 'op.pull-requests', session: 'codex-operator', probes: ['F2', 'F3', 'F4'], - text: 'Which pull requests did the session create? For each one, give its number, the time of the tool call that issued the create command, the time of the tool call that issued the successful merge command, and whether the session was shown a review of it before that merge.', - paraphrases: ['List every pull request this session opened. For each, report the PR number, when the session opened it and when it merged it (use the times of the tool calls that ran those commands), and whether any review of it was visible to the session before the merge.'], + text: 'Which pull requests did the session create? For each one, give its number, the time of the tool call that issued the create command, the time of the tool call that issued the successful merge command, and whether the session was shown a review of it before that merge. A review listing that came back empty is not a review.', + paraphrases: ['List every pull request this session opened. For each, report the PR number, when the session opened it and when it merged it (use the times of the tool calls that ran those commands), and whether any review of it was visible to the session before the merge, counting an empty review list as none.'], schema: { prs: { kind: 'records', diff --git a/bench/audit/score.test.ts b/bench/audit/score.test.ts index ab6728f..4f95537 100644 --- a/bench/audit/score.test.ts +++ b/bench/audit/score.test.ts @@ -197,6 +197,11 @@ describe('answers files', () => { const file = parseAnswersFile({ arm: 'baseline', answers: [{ question: 'op.runs', answer: null, cost_usd: 0.01, cost_basis: 'estimated' }] }) expect(file.arm).toBe('baseline') }) + + it('validates the file scoreArm is handed, not only the one the CLI reads', () => { + expect(() => scoreArm({ arm: 'unchecked', answers: [{ question: 'op.nope', answer: null }] }, bench.gold, index)) + .toThrow(/not a known question id/) + }) }) describe('arm scores', () => { diff --git a/bench/audit/score.ts b/bench/audit/score.ts index d4b27e9..2a6d290 100644 --- a/bench/audit/score.ts +++ b/bench/audit/score.ts @@ -299,7 +299,12 @@ function tally(rows: ReadonlyArray<{ row: AnswerRow; scored: ScoredAnswer }>): T } } -export function scoreArm(file: AnswersFile, gold: GoldAnswers, index: CitationIndex): ArmScore { +/** + * Score one answers file. The file is validated here rather than by the caller, so a + * malformed one reports every problem instead of crashing on the first bad row. + */ +export function scoreArm(input: unknown, gold: GoldAnswers, index: CitationIndex): ArmScore { + const file = parseAnswersFile(input) const scored = file.answers.map((row) => ({ row, scored: scoreAnswer(questionById(row.question)!, row, gold, index) })) const questions: QuestionScore[] = [] const notAttempted: ArmScore['notAttempted'] = []