diff --git a/CURRENT_STATE.md b/CURRENT_STATE.md index bd06e22..42269bf 100644 --- a/CURRENT_STATE.md +++ b/CURRENT_STATE.md @@ -3,28 +3,54 @@ ## Goal Benchmark whether slice-isolated coding agents generalize from test-driven repair to typecheck-driven repair using the same projection/edit loop. -## Active Slice -Typecheck benchmark surface is implemented and wired end-to-end. Three plumbing fixtures pass `tsc --noEmit` failure through existing projectors, and results save separately. Awaiting first live benchmark run. +## Status +Typecheck benchmark is implemented, run, and producing clean results across 6 tasks (3 L1 + 3 L2). Results are recorded in `results/benchmark-typecheck.json`. -## Inputs -- `tsc --noEmit` output from task directory -- benchmark config at `configs/benchmark-typecheck.json` +mini-SWE-agent integration is wired end-to-end: adapter, runner, config, and CLI flags verified against v2.2.7. Ready for first benchmark run with `configs/benchmark-with-swe-agent.json`. -## Expected Output -- parsed `TypeErrorAnchor` with file/line/column/code/message -- deterministic primary anchor selection -- separate typecheck benchmark results at `results/benchmark-typecheck.json` +## Benchmark Tasks + +### L1 (single-file, anchor file alone is sufficient) +- `wrong_return_type` (TS2322) — all ZCA agents pass in 1 step +- `missing_property` (TS2339) — all ZCA agents pass in 1 step +- `undefined_name` (TS2304) — all ZCA agents pass in 1 step + +### L2 (cross-file, fix requires context beyond the anchor file) +- `cross_file_return_type` (TS2322) — both ZCA agents pass; naive may be guessing +- `wrong_method_call` (TS2339) — both ZCA agents pass; naive may be guessing +- `unresolved_cross_import` (TS2304) — **clean naive/adaptive split**: naive FAIL, adaptive PASS + +## Latest Results (Sonnet 4) + +| Agent | Pass rate | Input tokens | +|---|---|---| +| Baseline | 0/6 | 3.9M in / 13.7k out | +| ZCA Naive | 5/6 | 8.4k in / 4.0k out | +| ZCA Adaptive | 6/6 | 5.9k in / 2.0k out | + +## Interpretation +- Projection vs baseline is strongly supported on the typecheck surface +- Typecheck generalization is strongly supported — same loops, same projectors, new signal +- Adaptive vs naive is cleanly supported by `unresolved_cross_import` +- Two of three L2 tasks are soft — naive passes by local guessing, not structural reasoning + +## Bug Fixed This Iteration +The naive projector crashed with ENOENT on re-projection when `selectPrimaryAnchor` returned null (unsupported error code after model edit). The fallback path hit `inferSourceFile`, which is test-specific and defaults to `src/index.ts`. Fixed by: +1. Adding anchor fallback: `primary?.file ?? anchors[0]?.file` in `ZCAAgent.ts` +2. Adding `existsSync` guards in both `projectFailureSlice.ts` and `adaptiveProjector.ts` ## In Scope - `src/runtime/tools/runTypeCheck.ts` — signal execution + anchor parsing + selection -- `src/agents/zca/ZCAAgent.ts` — signal routing, typecheck projector construction +- `src/agents/zca/ZCAAgent.ts` — signal routing, typecheck projector construction, anchor fallback - `src/agents/baseline/BaselineAgent.ts` — signal routing for baseline - `src/agents/zca/zcaLoop.ts` — parameterized verify/prompt (shared by both signals) - `src/agents/baseline/baselineLoop.ts` — parameterized verify/prompt (shared by both signals) -- `src/agents/zca/projectFailureSlice.ts` — entryFile override (no new projector) -- `src/agents/zca/adaptiveProjector.ts` — entryFile override (no new projector) -- `experiments/tasks-typecheck/` — 3 plumbing fixtures +- `src/agents/zca/projectFailureSlice.ts` — entryFile override + existsSync guard +- `src/agents/zca/adaptiveProjector.ts` — entryFile override + existsSync guard +- `src/agents/sweAgent/MiniSWEAgentAdapter.ts` — subprocess adapter for mini-SWE-agent CLI +- `experiments/tasks-typecheck/` — 6 fixtures (3 L1 + 3 L2) - `configs/benchmark-typecheck.json` — separate config +- `configs/benchmark-with-swe-agent.json` — test benchmark with mini-SWE-agent baseline ## Out of Scope - generic plugin/signal framework @@ -38,52 +64,20 @@ Typecheck benchmark surface is implemented and wired end-to-end. Three plumbing - supported error codes: TS2322, TS2339, TS2304 only - anchor selection ignores node_modules, dist, .d.ts files - one primary anchor per iteration (deterministic: sorted by file/line/code, first match) +- anchor fallback to first raw anchor when supported-code filter is empty - projectors reused directly via entryFile override — no separate typecheck projector files - loops reused directly via verify/prompt parameterization — no separate loop files -- 3 fixture tasks are plumbing-quality, not published benchmark coverage - -## Unknown Constraints -- whether Sonnet/Haiku can reliably fix type errors with the current prompt framing -- whether adaptive projector's import-following helps for type errors (may already be single-file) -- whether 3 tasks are enough to expose architectural differences between agents -- whether `tsc --noEmit` startup time affects duration metrics meaningfully - -## Verification -- Command: `npm run benchmark -- configs/benchmark-typecheck.json` -- Success condition: all 3 tasks run for all 3 agents, results save to `results/benchmark-typecheck.json` - -## Current Owners -- signal execution + anchor parsing → `src/runtime/tools/runTypeCheck.ts` -- anchor → projector routing → `src/agents/zca/ZCAAgent.ts` -- naive projection → `src/agents/zca/projectFailureSlice.ts` -- adaptive projection → `src/agents/zca/adaptiveProjector.ts` -- ZCA loop → `src/agents/zca/zcaLoop.ts` -- baseline loop → `src/agents/baseline/baselineLoop.ts` -- benchmark orchestration → `src/scripts/runBenchmark.ts` -- task sandboxing → `src/runtime/execution/sandbox.ts` -- result types → `src/analysis/metrics/types.ts` - -## Files Changed (this iteration) -- **new:** `src/runtime/tools/runTypeCheck.ts` -- **new:** `configs/benchmark-typecheck.json` -- **new:** `experiments/tasks-typecheck/wrong_return_type/` (TS2322) -- **new:** `experiments/tasks-typecheck/missing_property/` (TS2339) -- **new:** `experiments/tasks-typecheck/undefined_name/` (TS2304) -- **modified:** `src/agents/zca/ZCAAgent.ts` — signal routing, typecheck projector builder -- **modified:** `src/agents/zca/zcaLoop.ts` — optional verify/systemPrompt/goal/failureLabel -- **modified:** `src/agents/zca/projectFailureSlice.ts` — optional entryFile parameter -- **modified:** `src/agents/zca/adaptiveProjector.ts` — options object with entryFile -- **modified:** `src/agents/zca/canonicalizeState.ts` — optional goal/failureLabel -- **modified:** `src/agents/zca/zcaPrompt.ts` — uses failureLabel from state -- **modified:** `src/agents/baseline/BaselineAgent.ts` — signal routing -- **modified:** `src/agents/baseline/baselineLoop.ts` — optional verify/prompt/messages -- **modified:** `src/runtime/tools/index.ts` — signal-aware tool registry -- **modified:** `src/runtime/execution/taskPaths.ts` — optional tasksDir -- **modified:** `src/runtime/execution/sandbox.ts` — propagate tasksDir -- **modified:** `src/scripts/runBenchmark.ts` — parse signal/tasksDir from config -- **modified:** `src/analysis/metrics/types.ts` — typecheck task classifications -- **modified:** `src/runtime/execution/logger.ts` — verbose logging (from earlier) -- **modified:** `src/scripts/runZCA.ts` — --verbose flag (from earlier) + +## Resolved Questions +- **Will the baseline agent handle tsc errors?** No — same exploration-without-editing pattern as tests. +- **Does adaptive projector's import-following help for type errors?** Yes — cleanly demonstrated on `unresolved_cross_import`. +- **Are 3 tasks enough to expose differences?** No — 3 L1 tasks showed no naive/adaptive split. After adding 3 L2 tasks, one (`unresolved_cross_import`) cleanly separates them. +- **Is `tsc --noEmit` startup time a problem?** No — each verification takes ~1s, negligible vs model latency. + +## Open Questions +- How to design L2/L3 tasks where the naive projector reliably fails (not just sometimes) +- Whether `extractFailingSymbol` should be adapted for tsc output (currently test-oriented) +- Whether bounded parallel execution in the benchmark runner would meaningfully reduce wall-clock time ## Architecture Shape ``` @@ -119,11 +113,5 @@ Typecheck benchmark surface is implemented and wired end-to-end. Three plumbing └──────────────────────────────────┘ ``` -## Open Questions -- will the baseline agent's exploration loop handle `tsc` errors as effectively as test errors? -- should the typecheck prompt include the specific error code and line for better targeting? -- is `npx tsc --noEmit` startup latency acceptable or should we cache the compiler? -- should future tasks include multi-file type errors (L2/L3 locality)? - ## Next Minimal Step -Run the typecheck benchmark end-to-end with `npm run benchmark -- configs/benchmark-typecheck.json` and record results. Then assess whether the projection architecture holds or needs adjustment before adding more tasks. +Run the SWE-agent benchmark with `npm run benchmark -- configs/benchmark-with-swe-agent.json` and record results. Then design harder L2/L3 typecheck tasks where the naive projector reliably fails. diff --git a/README.md b/README.md index c397bde..9c6fc2a 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ How obvious the bug source is from the failure signal. # Agents compared -Three agents are evaluated. +Four agents are evaluated. ## Baseline agent @@ -122,6 +122,14 @@ Context grows across iterations. A "step" is counted only when the agent makes a --- +## mini-SWE-agent (external baseline) + +An external coding agent used as a second baseline for comparison. [mini-SWE-agent](https://github.com/SWE-agent/mini-swe-agent) is a compact autonomous software-engineering agent from the SWE-bench ecosystem. + +It runs the same tasks, in the same sandboxes, evaluated by the same verifiers. The benchmark harness invokes it as a subprocess and collects metrics externally. This provides a recognizable external reference point without changing the benchmark architecture. + +--- + ## ZCA Naive Slice-isolated execution with a simple projector. @@ -255,6 +263,15 @@ At the same time, some L2 tasks still appear soft or locally guessable, so the c --- +# Current interpretation + +- **Projection vs baseline** is strongly supported across both test and typecheck surfaces. The baseline agent consistently fails by exploring without editing, regardless of failure signal. +- **Typecheck generalization** is strongly supported. The same projection/edit loop, with no changes to projectors or agent loops, produces clean results on TypeScript compiler errors. +- **Adaptive vs naive** is now cleanly supported by `unresolved_cross_import`, where the naive projector fails across 5 steps (it only sees the anchor file) while the adaptive projector solves it in 1 step (it follows imports to include context). +- **More hard L2 tasks are still needed.** Two of three L2 tasks were solved by naive — likely because the model could guess the correct fix from the anchor file alone. Tasks where the fix is not locally inferrable are needed to further stress the naive/adaptive boundary. + +--- + # Key findings ### Execution architecture strongly affects agent behavior @@ -342,6 +359,15 @@ If you want to run the compiler-driven benchmark: npm run benchmark -- configs/benchmark-typecheck.json ``` +To run the benchmark including mini-SWE-agent as an external baseline: + +```bash +pip install mini-swe-agent +npm run benchmark -- configs/benchmark-with-swe-agent.json +``` + +mini-SWE-agent is a Python CLI. It must be installed separately and available as `mini` on your PATH. The adapter invokes it as a subprocess and does not require any other Python dependencies in this repository. + --- # Repository structure @@ -364,6 +390,7 @@ experiments/tasks-typecheck/ # compiler-driven benchmark task definit src/ agents/baseline/ # long-context baseline agent agents/zca/ # ZCA agent + naive/adaptive projectors + agents/sweAgent/ # mini-SWE-agent subprocess adapter model/ # model client abstraction runtime/ # sandbox execution and tool registry analysis/ # result types and metrics @@ -373,6 +400,7 @@ configs/ benchmark.json # same-model (Sonnet) test benchmark benchmark-cross-model.json # cross-model (Opus vs Haiku) test benchmark benchmark-typecheck.json # typecheck-driven benchmark + benchmark-with-swe-agent.json # test benchmark with mini-SWE-agent baseline results/ # benchmark output JSON files ``` diff --git a/configs/README.md b/configs/README.md index 20c15f4..19ca44d 100644 --- a/configs/README.md +++ b/configs/README.md @@ -9,4 +9,8 @@ Configuration files for agent runs and benchmark matrix. | `baseline.json` | Config for baseline agent (max steps, model provider) | | `zca.json` | Config for ZCA agent with naive projector | | `zca-adaptive.json` | Config for ZCA agent with adaptive projector | -| `benchmark.json` | Full benchmark matrix: tasks × agents × model | +| `benchmark.json` | Same-model (Sonnet) test benchmark: tasks × agents × model | +| `benchmark-cross-model.json` | Cross-model benchmark: Opus baseline vs Haiku ZCA agents | +| `benchmark-typecheck.json` | TypeScript compiler-driven benchmark (all 6 typecheck tasks) | +| `benchmark-typecheck-fast.json` | Typecheck benchmark subset (L2 tasks only, ZCA agents only) | +| `benchmark-with-swe-agent.json` | Test benchmark with mini-SWE-agent as external baseline | diff --git a/configs/benchmark-typecheck-fast.json b/configs/benchmark-typecheck-fast.json new file mode 100644 index 0000000..d203040 --- /dev/null +++ b/configs/benchmark-typecheck-fast.json @@ -0,0 +1,18 @@ +{ + "signal": "typecheck", + "tasksDir": "experiments/tasks-typecheck", + "tasks": [ + "cross_file_return_type", + "wrong_method_call", + "unresolved_cross_import" + ], + "agents": [ + { "name": "zca-naive", "type": "zca", "projector": "naive", "maxSteps": 5 }, + { "name": "zca-adaptive", "type": "zca", "projector": "adaptive", "maxSteps": 5 } + ], + "model": { + "provider": "anthropic", + "model": "claude-sonnet-4-20250514", + "temperature": 0 + } +} diff --git a/configs/benchmark-with-swe-agent.json b/configs/benchmark-with-swe-agent.json new file mode 100644 index 0000000..385afbf --- /dev/null +++ b/configs/benchmark-with-swe-agent.json @@ -0,0 +1,19 @@ +{ + "tasks": [ + "parser_bug", + "range_check_bug", + "slug_conflict_bug", + "config_lookup_bug" + ], + "agents": [ + { "name": "baseline", "type": "baseline", "maxSteps": 10 }, + { "name": "mini-swe-agent", "type": "swe", "maxSteps": 10, "costLimit": 2.0, "timeout": 300000 }, + { "name": "zca-naive", "type": "zca", "projector": "naive", "maxSteps": 5 }, + { "name": "zca-adaptive", "type": "zca", "projector": "adaptive", "maxSteps": 5 } + ], + "model": { + "provider": "anthropic", + "model": "claude-sonnet-4-20250514", + "temperature": 0 + } +} diff --git a/package-lock.json b/package-lock.json index 5cfd7d5..92550aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,8 @@ "name": "slice-agent-bench", "version": "0.1.0", "dependencies": { - "@anthropic-ai/sdk": "^0.79.0" + "@anthropic-ai/sdk": "^0.79.0", + "dotenv": "^17.3.1" }, "devDependencies": { "@types/node": "^25.5.0", @@ -497,6 +498,18 @@ "undici-types": "~7.18.0" } }, + "node_modules/dotenv": { + "version": "17.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.3.1.tgz", + "integrity": "sha512-IO8C/dzEb6O3F9/twg6ZLXz164a2fhTnEWb95H23Dm4OuN+92NmEAlTrupP9VW6Jm3sO26tQlqyvyi4CsnY9GA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/esbuild": { "version": "0.27.4", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.4.tgz", diff --git a/package.json b/package.json index 9cef5f4..4cf6d3e 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "typescript": "^5.4.0" }, "dependencies": { - "@anthropic-ai/sdk": "^0.79.0" + "@anthropic-ai/sdk": "^0.79.0", + "dotenv": "^17.3.1" } } diff --git a/results/benchmark-typecheck.json b/results/benchmark-typecheck.json index d93dd12..53cc0ca 100644 --- a/results/benchmark-typecheck.json +++ b/results/benchmark-typecheck.json @@ -1,6 +1,6 @@ { "results": [ - { + { "task": "wrong_return_type", "agent": "baseline", "success": false, diff --git a/results/benchmark-with-swe-agent.json b/results/benchmark-with-swe-agent.json new file mode 100644 index 0000000..0d6bc43 --- /dev/null +++ b/results/benchmark-with-swe-agent.json @@ -0,0 +1,20 @@ +{ + "results": [ + { + "task": "parser_bug", + "agent": "mini-swe-agent", + "success": true, + "steps": 0, + "durationMs": 120460, + "inputTokens": 0, + "outputTokens": 0 + } + ], + "matrix": [], + "tasks": [ + "parser_bug" + ], + "agents": [ + "mini-swe-agent" + ] +} \ No newline at end of file diff --git a/src/agents/sweAgent/MiniSWEAgentAdapter.ts b/src/agents/sweAgent/MiniSWEAgentAdapter.ts new file mode 100644 index 0000000..25525c4 --- /dev/null +++ b/src/agents/sweAgent/MiniSWEAgentAdapter.ts @@ -0,0 +1,291 @@ +import { spawn } from "node:child_process"; +import { readFile, unlink } from "node:fs/promises"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; +import { Logger } from "../../runtime/execution/logger.js"; +import { runTests } from "../../runtime/tools/runTests.js"; +import { runTypeCheck } from "../../runtime/tools/runTypeCheck.js"; +import type { SignalType } from "../zca/ZCAAgent.js"; + +const PROBLEM_STATEMENTS: Record = { + test: [ + "There is a failing test in this repository.", + "Fix the code so that the test suite passes.", + "Do not make unrelated changes.", + ].join("\n"), + typecheck: [ + "There is a failing TypeScript compiler error in this repository.", + "Fix the code so that `tsc --noEmit` passes.", + "Do not make unrelated changes.", + ].join("\n"), +}; + +export interface MiniSWEAgentConfig { + taskName: string; + taskPath: string; + signal: SignalType; + model: string; + costLimit?: number; + timeout?: number; +} + +export interface MiniSWEAgentResult { + success: boolean; + steps: number; + totalInputTokens: number; + totalOutputTokens: number; +} + +interface TrajectoryEntry { + role?: string; + content?: string; + tool_calls?: unknown[]; + usage?: { + input_tokens?: number; + output_tokens?: number; + prompt_tokens?: number; + completion_tokens?: number; + }; + actions?: unknown[]; +} + +export class MiniSWEAgentAdapter { + private readonly config: MiniSWEAgentConfig; + private readonly logger: Logger; + + constructor(config: MiniSWEAgentConfig) { + this.config = config; + this.logger = new Logger("mini-swe"); + } + + async run(): Promise { + const { taskPath, signal, model, taskName } = this.config; + const costLimit = this.config.costLimit ?? 2.0; + const timeout = this.config.timeout ?? 300_000; + + this.logger.info(`Task: ${taskName}`); + this.logger.info(`Task path: ${taskPath}`); + this.logger.info(`Model: ${model}`); + this.logger.info(`Cost limit: $${costLimit}`); + + const trajectoryPath = join(taskPath, `.trajectory-${randomUUID()}.json`); + const problemStatement = PROBLEM_STATEMENTS[signal]; + + try { + await this.invokeAgent(taskPath, problemStatement, model, costLimit, timeout, trajectoryPath); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + this.logger.error(`Agent invocation failed: ${msg}`); + } + + const { steps, inputTokens, outputTokens } = await this.parseTrajectory(trajectoryPath); + + const verifier = signal === "typecheck" ? runTypeCheck : runTests; + const verification = await verifier({ taskPath }); + const success = verification.success; + + this.logger.info(`Verification: ${success ? "PASS" : "FAIL"}`); + this.logger.info(`Steps: ${steps}, Tokens: ${inputTokens}in/${outputTokens}out`); + + try { + await unlink(trajectoryPath); + } catch { + // best-effort cleanup + } + + return { + success, + steps, + totalInputTokens: inputTokens, + totalOutputTokens: outputTokens, + }; + } + + private invokeAgent( + cwd: string, + task: string, + model: string, + costLimit: number, + timeout: number, + trajectoryPath: string, + ): Promise { + return new Promise((resolve, reject) => { + const args = [ + "-y", + "--exit-immediately", + "-t", task, + "-m", model, + "-l", String(costLimit), + "-o", trajectoryPath, + ]; + + this.logger.info(`Spawning: mini ${args.join(" ")}`); + + const proc = spawn("mini", args, { + cwd, + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + MSWEA_CONFIGURED: "1", + }, + }); + + let stdout = ""; + let stderr = ""; + + proc.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + + proc.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + const timer = setTimeout(() => { + this.logger.warn(`Timeout after ${timeout}ms, killing process`); + proc.kill("SIGTERM"); + setTimeout(() => { + if (!proc.killed) { + proc.kill("SIGKILL"); + } + }, 5_000); + }, timeout); + + proc.on("error", (err) => { + clearTimeout(timer); + if (err.message.includes("ENOENT")) { + reject(new Error( + "mini-swe-agent CLI not found. Install it with: pip install mini-swe-agent", + )); + return; + } + reject(err); + }); + + proc.on("close", (code) => { + clearTimeout(timer); + if (code !== 0 && stderr.trim()) { + this.logger.error(`mini stderr:\n${stderr.trim()}`); + } else if (stderr.trim()) { + this.logger.verbose("mini stderr", stderr.trim()); + } + if (stdout.trim()) { + this.logger.verbose("mini stdout", stdout.trim()); + } + this.logger.info(`mini exited with code ${code}`); + resolve(); + }); + }); + } + + private async parseTrajectory( + trajectoryPath: string, + ): Promise<{ steps: number; inputTokens: number; outputTokens: number }> { + try { + const raw = await readFile(trajectoryPath, "utf-8"); + const data = JSON.parse(raw); + this.logger.info(`Trajectory shape: ${describeShape(data)}`); + if (typeof data === "object" && data !== null) { + const obj = data as Record; + if (Array.isArray(obj["messages"])) { + const msgs = obj["messages"] as Record[]; + const roles = msgs.map((m) => m["role"]).filter(Boolean); + const withUsage = msgs.filter((m) => m["usage"]).length; + const withExtra = msgs.filter((m) => m["extra"]).length; + this.logger.info( + `Messages: ${msgs.length} total, roles=[${[...new Set(roles)].join(",")}], ` + + `${withUsage} with usage, ${withExtra} with extra`, + ); + if (msgs.length > 0) { + this.logger.info(`First message keys: [${Object.keys(msgs[0]).join(",")}]`); + const lastAssistant = [...msgs].reverse().find((m) => m["role"] === "assistant"); + if (lastAssistant) { + this.logger.info(`Last assistant keys: [${Object.keys(lastAssistant).join(",")}]`); + } + } + } + if (typeof obj["info"] === "object" && obj["info"] !== null) { + this.logger.info(`Info keys: [${Object.keys(obj["info"] as object).join(",")}]`); + } + } + return extractMetrics(data); + } catch { + this.logger.warn("Could not read trajectory file, returning zero metrics"); + return { steps: 0, inputTokens: 0, outputTokens: 0 }; + } + } +} + +function extractMetrics( + trajectory: unknown, +): { steps: number; inputTokens: number; outputTokens: number } { + let steps = 0; + let inputTokens = 0; + let outputTokens = 0; + + if (Array.isArray(trajectory)) { + for (const entry of trajectory) { + const e = entry as TrajectoryEntry; + + if (e.role === "assistant" || e.actions || e.tool_calls) { + steps++; + } + + if (e.usage) { + inputTokens += e.usage.input_tokens ?? e.usage.prompt_tokens ?? 0; + outputTokens += e.usage.output_tokens ?? e.usage.completion_tokens ?? 0; + } + } + } else if (typeof trajectory === "object" && trajectory !== null) { + const obj = trajectory as Record; + + if (Array.isArray(obj["history"])) { + for (const entry of obj["history"]) { + const e = entry as TrajectoryEntry; + if (e.role === "assistant" || e.actions || e.tool_calls) { + steps++; + } + if (e.usage) { + inputTokens += e.usage.input_tokens ?? e.usage.prompt_tokens ?? 0; + outputTokens += e.usage.output_tokens ?? e.usage.completion_tokens ?? 0; + } + } + } + + if (typeof obj["info"] === "object" && obj["info"] !== null) { + const info = obj["info"] as Record; + if (typeof info["total_cost"] === "number" && steps === 0) { + steps = 1; + } + } + + if (typeof obj["steps"] === "number") { + steps = obj["steps"] as number; + } + if (typeof obj["input_tokens"] === "number") { + inputTokens = obj["input_tokens"] as number; + } + if (typeof obj["output_tokens"] === "number") { + outputTokens = obj["output_tokens"] as number; + } + } + + return { steps, inputTokens, outputTokens }; +} + +function describeShape(data: unknown): string { + if (Array.isArray(data)) { + const sample = data[0]; + const keys = sample && typeof sample === "object" ? Object.keys(sample as object).join(",") : "?"; + return `array[${data.length}] first-keys=[${keys}]`; + } + if (typeof data === "object" && data !== null) { + const keys = Object.keys(data); + return `object keys=[${keys.join(",")}]`; + } + return typeof data; +} + +export function toLitellmModel(provider: string, model: string): string { + return `${provider}/${model}`; +} diff --git a/src/scripts/runBenchmark.ts b/src/scripts/runBenchmark.ts index 317ba47..662aa4b 100644 --- a/src/scripts/runBenchmark.ts +++ b/src/scripts/runBenchmark.ts @@ -1,8 +1,10 @@ +import "dotenv/config"; import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; import { BaselineAgent } from "../agents/baseline/BaselineAgent.js"; import { ZCAAgent } from "../agents/zca/ZCAAgent.js"; import type { ProjectorMode, SignalType } from "../agents/zca/ZCAAgent.js"; +import { MiniSWEAgentAdapter, toLitellmModel } from "../agents/sweAgent/MiniSWEAgentAdapter.js"; import { createModelClient, createModelFactory } from "../model/factory.js"; import type { ModelConfig } from "../model/factory.js"; import { Logger } from "../runtime/execution/logger.js"; @@ -12,11 +14,13 @@ import { TASK_CLASSIFICATIONS } from "../analysis/metrics/types.js"; interface AgentSpec { name: string; - type: "baseline" | "zca"; + type: "baseline" | "zca" | "swe"; projector?: ProjectorMode; maxSteps: number; model?: ModelConfig; signal?: SignalType; + costLimit?: number; + timeout?: number; } interface BenchmarkConfig { @@ -27,6 +31,62 @@ interface BenchmarkConfig { tasksDir?: string; } +interface Job { + task: string; + agentSpec: AgentSpec; + index: number; +} + +interface CLIOptions { + configPath: string; + tasks?: string[]; + agents?: string[]; + levels?: string[]; + heavyConcurrency: number; + lightConcurrency: number; +} + +function parseCLI(argv: string[]): CLIOptions { + const args = argv.slice(2); + let configPath = "configs/benchmark.json"; + let tasks: string[] | undefined; + let agents: string[] | undefined; + let levels: string[] | undefined; + let heavyConcurrency = 1; + let lightConcurrency = 1; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + const next = args[i + 1]; + + if (arg === "--tasks" && next) { + tasks = next.split(",").map((s) => s.trim()); + i++; + } else if (arg === "--agents" && next) { + agents = next.split(",").map((s) => s.trim()); + i++; + } else if (arg === "--levels" && next) { + levels = next.split(",").map((s) => s.trim().toUpperCase()); + i++; + } else if (arg === "--heavy-concurrency" && next) { + heavyConcurrency = Math.max(1, Number(next)); + i++; + } else if (arg === "--light-concurrency" && next) { + lightConcurrency = Math.max(1, Number(next)); + i++; + } else if (arg === "--concurrency" && next) { + const n = Math.max(1, Number(next)); + heavyConcurrency = n; + lightConcurrency = n; + i++; + } else if (!arg.startsWith("--")) { + configPath = arg; + } + } + + return { configPath, tasks, agents, levels, heavyConcurrency, lightConcurrency }; +} + function parseConfig(raw: unknown): BenchmarkConfig { if (typeof raw !== "object" || raw === null) { throw new Error("Invalid config: expected an object"); @@ -57,7 +117,7 @@ function parseConfig(raw: unknown): BenchmarkConfig { } return { name: String(spec["name"] ?? "unknown"), - type: String(spec["type"] ?? "baseline") as "baseline" | "zca", + type: String(spec["type"] ?? "baseline") as "baseline" | "zca" | "swe", projector: typeof spec["projector"] === "string" ? (spec["projector"] as ProjectorMode) : undefined, @@ -66,6 +126,12 @@ function parseConfig(raw: unknown): BenchmarkConfig { signal: typeof spec["signal"] === "string" ? (spec["signal"] as SignalType) : undefined, + costLimit: typeof spec["costLimit"] === "number" + ? spec["costLimit"] + : undefined, + timeout: typeof spec["timeout"] === "number" + ? spec["timeout"] + : undefined, }; }); @@ -131,6 +197,28 @@ async function runOneAgent( }; } + if (agentSpec.type === "swe") { + const litellmModel = toLitellmModel(modelConfig.provider, modelConfig.model); + const agent = new MiniSWEAgentAdapter({ + taskName, + taskPath: sandbox.workPath, + signal, + model: litellmModel, + costLimit: agentSpec.costLimit, + timeout: agentSpec.timeout, + }); + const result = await agent.run(); + return { + task: taskName, + agent: agentSpec.name, + success: result.success, + steps: result.steps, + durationMs: Date.now() - startMs, + inputTokens: result.totalInputTokens, + outputTokens: result.totalOutputTokens, + }; + } + const factory = createModelFactory(modelConfig); const agent = new ZCAAgent({ taskName, @@ -155,6 +243,29 @@ async function runOneAgent( } } +async function runPool( + items: (() => Promise)[], + concurrency: number, +): Promise { + const results: T[] = new Array(items.length); + let nextIndex = 0; + + async function worker(): Promise { + while (nextIndex < items.length) { + const i = nextIndex++; + results[i] = await items[i](); + } + } + + await Promise.all( + Array.from( + { length: Math.min(concurrency, items.length) }, + () => worker(), + ), + ); + return results; +} + function formatTokens(n: number): string { if (n >= 1_000_000) { return `${(n / 1_000_000).toFixed(1)}M`; @@ -210,7 +321,8 @@ function printMatrix(summary: BenchmarkSummary): void { const wins = summary.results.filter( (r) => r.agent === agent && r.success, ).length; - totalRow += `${wins}/${summary.tasks.length}`.padEnd(colWidth); + const total = summary.results.filter((r) => r.agent === agent).length; + totalRow += `${wins}/${total}`.padEnd(colWidth); } console.log(totalRow); @@ -227,60 +339,126 @@ function printMatrix(summary: BenchmarkSummary): void { console.log(); } +function buildJobs(config: BenchmarkConfig, cli: CLIOptions): Job[] { + let { tasks } = config; + let agents = config.agents; + + if (cli.tasks) { + const allowed = new Set(cli.tasks); + tasks = tasks.filter((t) => allowed.has(t)); + } + + if (cli.levels) { + const allowed = new Set(cli.levels); + tasks = tasks.filter((t) => { + const cls = TASK_CLASSIFICATIONS[t]; + return cls && allowed.has(cls.locality); + }); + } + + if (cli.agents) { + const allowed = new Set(cli.agents); + agents = agents.filter((a) => allowed.has(a.name)); + } + + const jobs: Job[] = []; + let index = 0; + for (const task of tasks) { + for (const agentSpec of agents) { + jobs.push({ task, agentSpec, index: index++ }); + } + } + return jobs; +} + async function main(): Promise { const logger = new Logger("benchmark"); + const cli = parseCLI(process.argv); - const configPath = process.argv[2] ?? "configs/benchmark.json"; - logger.info(`Loading benchmark config from ${configPath}`); + logger.info(`Loading benchmark config from ${cli.configPath}`); - const raw = await readFile(resolve(configPath), "utf-8"); + const raw = await readFile(resolve(cli.configPath), "utf-8"); const config = parseConfig(JSON.parse(raw)); - logger.info(`Tasks: ${config.tasks.join(", ")}`); - logger.info(`Agents: ${config.agents.map((a) => a.name).join(", ")}`); + const allJobs = buildJobs(config, cli); + + const filteredTasks = [...new Set(allJobs.map((j) => j.task))]; + const filteredAgents = [...new Set(allJobs.map((j) => j.agentSpec.name))]; + + logger.info(`Tasks: ${filteredTasks.join(", ")}`); + logger.info(`Agents: ${filteredAgents.join(", ")}`); + logger.info(`Jobs: ${allJobs.length}`); logger.info(`Default model: ${config.model.provider}/${config.model.model}`); - const results: BenchmarkResult[] = []; + const isHeavy = (j: Job): boolean => + j.agentSpec.type === "baseline" || j.agentSpec.type === "swe"; + const heavyJobs = allJobs.filter(isHeavy); + const lightJobs = allJobs.filter((j) => !isHeavy(j)); + + if (heavyJobs.length > 0 && lightJobs.length > 0) { + logger.info( + `Weighted concurrency: heavy=${cli.heavyConcurrency} (${heavyJobs.length} jobs), ` + + `light=${cli.lightConcurrency} (${lightJobs.length} jobs)`, + ); + } else { + const total = heavyJobs.length + lightJobs.length; + const c = heavyJobs.length > 0 ? cli.heavyConcurrency : cli.lightConcurrency; + logger.info(`Concurrency: ${c} (${total} jobs)`); + } + + function makeRunner(job: Job): () => Promise<{ job: Job; result: BenchmarkResult }> { + return async () => { + const { task, agentSpec } = job; + const tag = `${agentSpec.name}:${task}`; - for (const task of config.tasks) { - for (const agentSpec of config.agents) { - console.log(); - logger.info(`${"─".repeat(50)}`); - logger.info(`Running: ${agentSpec.name} on ${task}`); - logger.info(`${"─".repeat(50)}`); + logger.info(`▶ Starting ${tag}`); + const startWall = Date.now(); try { const result = await runOneAgent( task, agentSpec, config.model, logger, config.signal, config.tasksDir, ); - results.push(result); + const wallSec = ((Date.now() - startWall) / 1000).toFixed(1); logger.info( - `Done: ${result.success ? "PASS" : "FAIL"} in ${result.steps} steps ` + - `(${(result.durationMs / 1000).toFixed(1)}s, ` + - `${formatTokens(result.inputTokens)}in/${formatTokens(result.outputTokens)}out)`, + `✔ ${tag}: ${result.success ? "PASS" : "FAIL"} in ${result.steps} steps ` + + `(${wallSec}s, ${formatTokens(result.inputTokens)}in/${formatTokens(result.outputTokens)}out)`, ); + return { job, result }; } catch (error) { - logger.error( - `Error running ${agentSpec.name} on ${task}: ${error}`, - ); - results.push({ - task, - agent: agentSpec.name, - success: false, - steps: 0, - durationMs: 0, - inputTokens: 0, - outputTokens: 0, - }); + logger.error(`✘ ${tag}: ${error}`); + return { + job, + result: { + task, + agent: agentSpec.name, + success: false, + steps: 0, + durationMs: 0, + inputTokens: 0, + outputTokens: 0, + }, + }; } - } + }; } + const heavyRunners = heavyJobs.map(makeRunner); + const lightRunners = lightJobs.map(makeRunner); + + const [heavyResults, lightResults] = await Promise.all([ + runPool(heavyRunners, cli.heavyConcurrency), + runPool(lightRunners, cli.lightConcurrency), + ]); + + const allResults = [...heavyResults, ...lightResults] + .sort((a, b) => a.job.index - b.job.index) + .map((r) => r.result); + const summary: BenchmarkSummary = { - results, + results: allResults, matrix: [], - tasks: config.tasks, - agents: config.agents.map((a) => a.name), + tasks: filteredTasks, + agents: filteredAgents, }; printMatrix(summary); @@ -289,12 +467,12 @@ async function main(): Promise { const { basename } = await import("node:path"); await mkdir(resolve("results"), { recursive: true }); - const configName = basename(configPath, ".json"); + const configName = basename(cli.configPath, ".json"); const namedPath = resolve(`results/${configName}.json`); await writeFile(namedPath, JSON.stringify(summary, null, 2)); logger.info(`Results saved to ${namedPath}`); - const allPassed = results.every((r) => r.success); + const allPassed = allResults.every((r) => r.success); process.exit(allPassed ? 0 : 1); }