diff --git a/benchmarks/RUBRIC.md b/benchmarks/RUBRIC.md index 7532089f..44f9e2b1 100644 --- a/benchmarks/RUBRIC.md +++ b/benchmarks/RUBRIC.md @@ -257,6 +257,28 @@ SWE_MODEL=openai/gpt-5 \ grades only the last one → **grade per model separately**. - Image tag: `swebench/sweb.eval.x86_64.`. +### Verified N=30 cohort (AGT-3185) + +`benchmarks/sweBenchVerifiedSet.ts` generates a fixed N=30 cohort: ten each +from pylint, Sphinx, and SymPy. These pure-logic repositories avoid the known +external-http dependency in old `requests` cases, while fixing the denominator +before comparing models. + +```bash +# Writes the complete official records needed by sweBench.ts. +npx tsx benchmarks/sweBenchVerifiedSet.ts --out /tmp/swebench-verified-n30.json + +# Low-disk preflight: prove all 30 tags resolve without storing image layers. +npx tsx benchmarks/sweBenchVerifiedSet.ts --check-manifests --out /tmp/swebench-verified-n30.json + +# Only after confirming Docker storage headroom: sequentially pull all amd64 images. +npx tsx benchmarks/sweBenchVerifiedSet.ts --pull --out /tmp/swebench-verified-n30.json +``` + +The final command is intentionally opt-in: 30 evaluation images can exceed a +developer machine's free Docker storage. A successful manifest check is a tag +availability preflight, not a substitute for the ticket's required pull proof. + ## Harness defects — found and fixed at L6 (invisible on synthetic L0–L5) L6 exposed defects that only manifest in large repos: diff --git a/benchmarks/sweBenchVerifiedSet.ts b/benchmarks/sweBenchVerifiedSet.ts new file mode 100644 index 00000000..a4d578c4 --- /dev/null +++ b/benchmarks/sweBenchVerifiedSet.ts @@ -0,0 +1,163 @@ +#!/usr/bin/env tsx +// Generate the fixed, pure-logic SWE-bench Verified N=30 cohort used for +// worker-model comparison. The official dataset is the source of the complete +// problem/test records; this file deliberately stores only the stable IDs. + +import { execFile } from 'node:child_process'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); +const DATASET = 'SWE-bench/SWE-bench_Verified'; +const PAGE_SIZE = 100; +const DATASET_URL = 'https://datasets-server.huggingface.co/rows'; + +export const COHORT_REPOS = ['pylint-dev/pylint', 'sphinx-doc/sphinx', 'sympy/sympy'] as const; + +/** + * Fixed rather than "first 30 returned" so a dataset-server page-order change + * cannot silently change the denominator of a model comparison. + */ +export const COHORT_IDS = [ + 'pylint-dev__pylint-4551', + 'pylint-dev__pylint-4604', + 'pylint-dev__pylint-4661', + 'pylint-dev__pylint-4970', + 'pylint-dev__pylint-6386', + 'pylint-dev__pylint-6528', + 'pylint-dev__pylint-6903', + 'pylint-dev__pylint-7080', + 'pylint-dev__pylint-7277', + 'pylint-dev__pylint-8898', + 'sphinx-doc__sphinx-10323', + 'sphinx-doc__sphinx-10435', + 'sphinx-doc__sphinx-10449', + 'sphinx-doc__sphinx-10466', + 'sphinx-doc__sphinx-10614', + 'sphinx-doc__sphinx-10673', + 'sphinx-doc__sphinx-11445', + 'sphinx-doc__sphinx-11510', + 'sphinx-doc__sphinx-7440', + 'sphinx-doc__sphinx-7454', + 'sympy__sympy-11618', + 'sympy__sympy-12096', + 'sympy__sympy-12419', + 'sympy__sympy-12481', + 'sympy__sympy-12489', + 'sympy__sympy-13031', + 'sympy__sympy-13091', + 'sympy__sympy-13372', + 'sympy__sympy-13480', + 'sympy__sympy-13551', +] as const; + +export interface SweBenchInstance { + instance_id: string; + repo: string; + base_commit: string; + problem_statement: string; + FAIL_TO_PASS: string; + test_patch?: string; + [key: string]: unknown; +} + +interface DatasetPage { + rows?: Array<{ row?: SweBenchInstance }>; +} + +export function imageFor(instanceId: string): string { + return `swebench/sweb.eval.x86_64.${instanceId.replace('__', '_1776_')}:latest`; +} + +export function selectCohort(rows: SweBenchInstance[]): SweBenchInstance[] { + const byId = new Map(rows.map((row) => [row.instance_id, row])); + const cohort = COHORT_IDS.map((id) => byId.get(id)); + const missing = COHORT_IDS.filter((id, index) => !cohort[index]); + if (missing.length) throw new Error(`Official dataset did not return ${missing.length} cohort record(s): ${missing.join(', ')}`); + + const selected = cohort as SweBenchInstance[]; + for (const row of selected) { + if (!COHORT_REPOS.includes(row.repo as (typeof COHORT_REPOS)[number])) { + throw new Error(`Cohort member ${row.instance_id} unexpectedly belongs to ${row.repo}`); + } + for (const field of ['base_commit', 'problem_statement', 'FAIL_TO_PASS'] as const) { + if (!row[field]) throw new Error(`Cohort member ${row.instance_id} is missing required field ${field}`); + } + } + return selected; +} + +async function fetchPage(offset: number): Promise { + const url = new URL(DATASET_URL); + url.searchParams.set('dataset', DATASET); + url.searchParams.set('config', 'default'); + url.searchParams.set('split', 'test'); + url.searchParams.set('offset', String(offset)); + url.searchParams.set('length', String(PAGE_SIZE)); + const response = await fetch(url, { signal: AbortSignal.timeout(30_000) }); + if (!response.ok) throw new Error(`Dataset request failed at offset ${offset}: HTTP ${response.status}`); + const body = await response.json() as DatasetPage; + return (body.rows ?? []).flatMap(({ row }) => row ? [row] : []); +} + +export async function fetchCohort(): Promise { + const rows: SweBenchInstance[] = []; + for (let offset = 0; ; offset += PAGE_SIZE) { + const page = await fetchPage(offset); + rows.push(...page); + if (page.length < PAGE_SIZE) break; + } + return selectCohort(rows); +} + +async function dockerImageCheck(image: string, pull: boolean): Promise { + const args = pull ? ['pull', '--platform', 'linux/amd64', image] : ['manifest', 'inspect', image]; + await execFileAsync('docker', args, { maxBuffer: 1024 * 1024 * 8 }); +} + +function parseArgs(argv: string[]): { output: string; checkManifests: boolean; pull: boolean } { + let output = 'benchmarks/fixtures/swebench-verified-n30.json'; + let checkManifests = false; + let pull = false; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--out') { + output = argv[++index] ?? ''; + if (!output) throw new Error('--out requires a path'); + } else if (arg === '--check-manifests') { + checkManifests = true; + } else if (arg === '--pull') { + pull = true; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + return { output, checkManifests, pull }; +} + +async function main(): Promise { + const { output, checkManifests, pull } = parseArgs(process.argv.slice(2)); + const cohort = await fetchCohort(); + const destination = resolve(output); + await mkdir(dirname(destination), { recursive: true }); + await writeFile(destination, `${JSON.stringify(cohort, null, 2)}\n`); + console.log(`Wrote ${cohort.length} official ${DATASET} records to ${destination}`); + + if (checkManifests || pull) { + const mode = pull ? 'pull' : 'manifest'; + for (const row of cohort) { + const image = imageFor(row.instance_id); + process.stdout.write(`[${mode}] ${image}\n`); + await dockerImageCheck(image, pull); + } + console.log(`${mode} check passed for ${cohort.length} images`); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +}