From c474b2871aee658b496b4544c3ffc8ed8564021a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 13:19:47 +0000 Subject: [PATCH 1/5] feat(bench): agent A/B harness for cgis vs no cgis (#543) Runner, scoring and guard for a headless `claude -p` benchmark that asks the same code questions with and without the cgis MCP server. - cgis.bench.agent_task: task YAML, hand-checked answer keys, deterministic recall/precision from the answer's closing JSON block. - cgis.bench.transcript: stream-json parser; tool calls, reads, cost, residual context, plus sufficiency and allocation for #220. - cgis.bench.guard: PreToolUse hook blocking the cgis CLI, uv, sqlite3 and graph files in both arms; the same predicate flags contaminated runs. - scripts/agent_ab.py: fresh worktree per run, `run` (with --dry-run) and `report`. Not in CI: every real run is a paid session. - Four cgis tasks (impact, flow, orientation, negative control). Re-measured the self.* placeholder band (66 on main, 69 here: three str/dict receivers) and raised the test-count baseline to 2943. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L7RD9DDX3xidUCfgBEtRDa --- .github/test-count-baseline | 2 +- benchmarks/agent_ab/.gitignore | 3 + benchmarks/agent_ab/README.md | 84 ++++ .../tasks/cgis-control-sqlite-pragmas.yaml | 20 + .../tasks/cgis-flow-ingest-to-sqlite.yaml | 50 +++ .../tasks/cgis-impact-language-for.yaml | 27 ++ .../cgis-orientation-call-resolution.yaml | 36 ++ scripts/agent_ab.py | 397 ++++++++++++++++++ src/cgis/bench/__init__.py | 1 + src/cgis/bench/agent_task.py | 241 +++++++++++ src/cgis/bench/guard.py | 64 +++ src/cgis/bench/transcript.py | 284 +++++++++++++ .../self_parsing/test_receiver_resolution.py | 7 +- tests/unit/agent_ab_stubs.py | 110 +++++ tests/unit/test_agent_ab_guard.py | 88 ++++ tests/unit/test_agent_ab_script.py | 251 +++++++++++ tests/unit/test_agent_ab_task.py | 206 +++++++++ tests/unit/test_agent_ab_transcript.py | 125 ++++++ 18 files changed, 1994 insertions(+), 2 deletions(-) create mode 100644 benchmarks/agent_ab/.gitignore create mode 100644 benchmarks/agent_ab/README.md create mode 100644 benchmarks/agent_ab/tasks/cgis-control-sqlite-pragmas.yaml create mode 100644 benchmarks/agent_ab/tasks/cgis-flow-ingest-to-sqlite.yaml create mode 100644 benchmarks/agent_ab/tasks/cgis-impact-language-for.yaml create mode 100644 benchmarks/agent_ab/tasks/cgis-orientation-call-resolution.yaml create mode 100644 scripts/agent_ab.py create mode 100644 src/cgis/bench/__init__.py create mode 100644 src/cgis/bench/agent_task.py create mode 100644 src/cgis/bench/guard.py create mode 100644 src/cgis/bench/transcript.py create mode 100644 tests/unit/agent_ab_stubs.py create mode 100644 tests/unit/test_agent_ab_guard.py create mode 100644 tests/unit/test_agent_ab_script.py create mode 100644 tests/unit/test_agent_ab_task.py create mode 100644 tests/unit/test_agent_ab_transcript.py diff --git a/.github/test-count-baseline b/.github/test-count-baseline index 024c51cb..6dabc8c3 100644 --- a/.github/test-count-baseline +++ b/.github/test-count-baseline @@ -1 +1 @@ -2775 +2943 diff --git a/benchmarks/agent_ab/.gitignore b/benchmarks/agent_ab/.gitignore new file mode 100644 index 00000000..a9cd3abc --- /dev/null +++ b/benchmarks/agent_ab/.gitignore @@ -0,0 +1,3 @@ +# Raw stream-json sessions: large, and they quote repository source verbatim +# (owner-api is private). results.jsonl keeps the scored line for each. +transcripts/ diff --git a/benchmarks/agent_ab/README.md b/benchmarks/agent_ab/README.md new file mode 100644 index 00000000..046b7396 --- /dev/null +++ b/benchmarks/agent_ab/README.md @@ -0,0 +1,84 @@ +# Agent A/B: the same questions with and without cgis (#543) + +Does an agent answer a code question better, or cheaper, when the cgis MCP +server is connected? This directory holds the questions, their answer keys and +the results. The runner is `scripts/agent_ab.py`; parsing and scoring live in +`src/cgis/bench/` and are unit-tested on recorded transcripts, so the harness +can change without spending anything. + +Design notes and cost estimates: `research/543.md` in the project files. + +## What is measured + +**Primary, fixed before the first paid run:** + +- `recall`: the share of the answer key the answer names (symbols, files and + literal facts, pooled). `precision`: the share of what it names that is in + the key or in its `allowed_*` lists. +- `cost_usd`: `total_cost_usd` from the session's closing event. + +The hypothesis is that the `cgis` arm reaches at least the control arm's recall +for no more cost on `impact` and `flow` questions, and that `control`-type +questions show what cgis costs when it cannot help. + +**Secondary:** tool calls by name, distinct files read and bytes read, turns, +wall time, token usage, and the context the session ends holding +(`residual_context`). + +**For #220, not for ranking arms:** `sufficiency` (what the agent did right +after each cgis answer) and `allocation` (the share of files cgis named that +the final answer relied on). Definitions are in the docstring of +`src/cgis/bench/transcript.py`. + +The answer key is written from the code and checked by hand, never taken from +the cgis graph: a key derived from the graph would agree with every edge the +resolver gets wrong. Each task's `notes` say how it was checked. + +## Arms + +| arm | MCP | plugin skills | graph files | cgis / uv on PATH | +|---|---|---|---|---| +| `control` | none (`--strict-mcp-config`, empty config) | no | all deleted | no | +| `cgis` | this checkout's `cgis-mcp` | yes (the plugin minus its `.mcp.json`) | `graph.db` built before the clock starts | no | + +Both arms run under `cgis.bench.guard` as a PreToolUse hook, which refuses the +cgis CLI, uv, sqlite3 and any read of `graph.db`/`graph.json` through Bash or +the file tools. Without it the control arm is not a control: codegraph's own +benchmark caught its control agent calling their CLI through Bash in 26 of 28 +runs. The same predicate marks a finished run `contaminated` if a blocked call +ever returned output; such runs are counted in the report and left out of the +medians. + +Both arms allow `Read`, `Grep`, `Glob` and `Bash` (plus `mcp__cgis` in the +treatment arm, which has no server in control) and refuse edits, web access and +sub-agents, under `--permission-mode dontAsk`. Each session starts in a fresh +detached worktree at the task's pinned commit, with `--no-session-persistence` +and `--setting-sources project`, so user settings and earlier sessions do not +reach it. + +## Running + +```bash +uv run python scripts/agent_ab.py run --dry-run # commands only, spends nothing +uv run python scripts/agent_ab.py run --task cgis-impact-language-for --runs 2 +uv run python scripts/agent_ab.py run --repo owner-api=../ownima-backend +uv run python scripts/agent_ab.py report +``` + +`--model` defaults to `claude-sonnet-5-5`; `--max-budget-usd` (default 2.00) +caps each session. Every non-dry run is a paid session, so this never runs in +CI. Transcripts are written to `transcripts/` (git-ignored); `results.jsonl` +gets one line per (task, arm, run) and is meant to be committed with the write-up. + +## Tasks + +| id | type | repo | +|---|---|---| +| `cgis-impact-language-for` | impact | cgis | +| `cgis-flow-ingest-to-sqlite` | flow | cgis | +| `cgis-orientation-call-resolution` | orientation | cgis | +| `cgis-control-sqlite-pragmas` | control | cgis | + +This repository's `CLAUDE.md` describes its architecture in some detail. It is +the same in both arms, but it lowers what a graph can add here, which is one +reason owner-api and httpx come next. diff --git a/benchmarks/agent_ab/tasks/cgis-control-sqlite-pragmas.yaml b/benchmarks/agent_ab/tasks/cgis-control-sqlite-pragmas.yaml new file mode 100644 index 00000000..71ee8eff --- /dev/null +++ b/benchmarks/agent_ab/tasks/cgis-control-sqlite-pragmas.yaml @@ -0,0 +1,20 @@ +id: cgis-control-sqlite-pragmas +repo: cgis +sha: 715cd9a2ecee47949651344757edb4c7d09f64d7 +src_root: src +type: control +question: | + Which SQLite journal mode and busy timeout does `SQLiteStore` configure when it + opens a database, and in which method? +gold: + symbols: + - SQLiteStore.__init__ + files: + - src/cgis/storage/sqlite_store.py + facts: + - WAL + - "5000" +notes: | + Negative control: one file, one method, no cross-file structure. A graph should + not help here; this measures what cgis costs when it is not needed. + sqlite_store.py:83-84. diff --git a/benchmarks/agent_ab/tasks/cgis-flow-ingest-to-sqlite.yaml b/benchmarks/agent_ab/tasks/cgis-flow-ingest-to-sqlite.yaml new file mode 100644 index 00000000..789ce0f2 --- /dev/null +++ b/benchmarks/agent_ab/tasks/cgis-flow-ingest-to-sqlite.yaml @@ -0,0 +1,50 @@ +id: cgis-flow-ingest-to-sqlite +repo: cgis +sha: 715cd9a2ecee47949651344757edb4c7d09f64d7 +src_root: src +type: flow +question: | + Trace what happens when `cgis ingest --output graph.db` runs: starting + from the CLI command, list in order the functions and methods on the main path + that parse source files, resolve calls, and write nodes and edges to SQLite. +gold: + symbols: + - cli.ingest + - IngestionPipeline.run + - IngestionPipeline._process_file + - id: extractor.parse + any_of: [BaseExtractor.parse, PythonExtractor.parse, TypeScriptExtractor.parse] + - ResolverEngine.resolve + - IngestionPipeline._persist_incremental + - SQLiteStore.save_incremental_batch + files: + - src/cgis/cli.py + - src/cgis/pipeline.py + - src/cgis/resolver/engine.py + - src/cgis/storage/sqlite_store.py + allowed_symbols: + - IngestionPipeline.workspace_root + - SQLiteStore.record_ingest + - SQLiteStore.record_ingest_options + - build_extractors + - IngestionPipeline._get_extractor + - ImportNameCollector.collect + - SemanticUpliftEngine.execute_uplift + - SymbolResolver + - IndexBuilder.build + allowed_files: + - src/cgis/extractors/base.py + - src/cgis/extractors/python_extractor.py + - src/cgis/extractors/typescript_extractor.py + - src/cgis/extractors/registry.py + - src/cgis/resolver/symbols.py + - src/cgis/resolver/indices.py + - src/cgis/resolver/uplift.py + - src/cgis/import_names.py +notes: | + Read 2026-10-02: cli.ingest (cli.py:174) opens SQLiteStore and calls + IngestionPipeline.run(store=..., rebuild=True); run calls _process_file per file, + which calls the extractor's parse; then ResolverEngine.resolve (pipeline.py:213); + then _persist_incremental (pipeline.py:237), which calls + SQLiteStore.save_incremental_batch (pipeline.py:411). Order is not scored, only + presence. diff --git a/benchmarks/agent_ab/tasks/cgis-impact-language-for.yaml b/benchmarks/agent_ab/tasks/cgis-impact-language-for.yaml new file mode 100644 index 00000000..67bb4ce1 --- /dev/null +++ b/benchmarks/agent_ab/tasks/cgis-impact-language-for.yaml @@ -0,0 +1,27 @@ +id: cgis-impact-language-for +repo: cgis +sha: 715cd9a2ecee47949651344757edb4c7d09f64d7 +src_root: src +type: impact +question: | + In this repository, the function `language_for` in `src/cgis/extractors/registry.py` + is about to change its signature. List every function or method that calls it + directly, so each call site can be updated. +gold: + symbols: + - cli.structure + - registry.is_supported + - SourceCollector.collect_full_files + - GraphContextCollector.sections + - mermaid._node_slug + files: + - src/cgis/cli.py + - src/cgis/extractors/registry.py + - src/cgis/guardian/collector.py + - src/cgis/query/render/mermaid.py + allowed_symbols: + - registry.language_for +notes: | + Checked 2026-10-02 by grep (`language_for(` outside imports) and by reading each + enclosing def: cli.py:843, registry.py:88, collector.py:175 and :214, + mermaid.py:77. The cgis graph agrees, but the key was not taken from it. diff --git a/benchmarks/agent_ab/tasks/cgis-orientation-call-resolution.yaml b/benchmarks/agent_ab/tasks/cgis-orientation-call-resolution.yaml new file mode 100644 index 00000000..1b5d2920 --- /dev/null +++ b/benchmarks/agent_ab/tasks/cgis-orientation-call-resolution.yaml @@ -0,0 +1,36 @@ +id: cgis-orientation-call-resolution +repo: cgis +sha: 715cd9a2ecee47949651344757edb4c7d09f64d7 +src_root: src +type: orientation +question: | + Which files implement turning unresolved call targets (`raw_call:` edges + emitted by the extractors) into fully qualified names, and what does each of + them contribute? +gold: + symbols: + - ResolverEngine + - SymbolResolver + - id: indices + any_of: [IndexBuilder, SymbolIndex] + files: + - src/cgis/resolver/engine.py + - src/cgis/resolver/symbols.py + - src/cgis/resolver/indices.py + allowed_symbols: + - ResolverEngine.resolve + - ResolverEngine._resolved_call_edge + - js_builtin_target + - IngestionPipeline.run + - Edge + allowed_files: + - src/cgis/resolver/js_builtins.py + - src/cgis/core/models.py + - src/cgis/pipeline.py + - src/cgis/extractors/python_extractor.py + - src/cgis/extractors/typescript_extractor.py + - src/cgis/extractors/base.py +notes: | + Read 2026-10-02: engine.py drives resolution, symbols.py holds the strategies + over a SymbolIndex, indices.py builds the indices (module docstrings say so). + js_builtins.py and models.py are correct but optional. diff --git a/scripts/agent_ab.py b/scripts/agent_ab.py new file mode 100644 index 00000000..311e469d --- /dev/null +++ b/scripts/agent_ab.py @@ -0,0 +1,397 @@ +"""Agent A/B benchmark: the same code questions with and without cgis (#543). + +Usage: + uv run python scripts/agent_ab.py run --dry-run # print commands, spend nothing + uv run python scripts/agent_ab.py run --task cgis-impact-language-for --runs 2 + uv run python scripts/agent_ab.py run --repo owner-api=../owner-api --arm cgis + uv run python scripts/agent_ab.py report + +Each run is a headless `claude -p` in a fresh detached worktree at the task's +pinned commit, so no run sees another's files or a graph it did not build. + +- `control`: no MCP servers; any committed or stray graph file is deleted. +- `cgis`: the cgis MCP server from *this* checkout, the plugin's skills, and a + graph built by this checkout's `cgis ingest` before the clock starts. + +Both arms run under the same PreToolUse hook (`cgis.bench.guard`), the same +allowed tools (Read, Grep, Glob, Bash; no edits, no web, no sub-agents) and a +PATH with no cgis or uv on it. Costs money: every non-dry run is a real session. +Results append to `benchmarks/agent_ab/results.jsonl`, one line per +(task, arm, run); the raw stream-json transcripts go to `benchmarks/agent_ab/ +transcripts/`, which is git-ignored. +""" + +import argparse +import json +import os +import shutil +import statistics +import subprocess +import sys +import tempfile +import time +from collections import defaultdict +from collections.abc import Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Literal + +from cgis.bench.agent_task import AgentTask, extract_answer, load_tasks, score_answer +from cgis.bench.transcript import parse_transcript, run_metrics + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from guardian_replay_skeptic import worktree_at + +Arm = Literal["control", "cgis"] +ARMS: tuple[Arm, ...] = ("control", "cgis") + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_BENCH_DIR = _REPO_ROOT / "benchmarks" / "agent_ab" +_BIN = Path(sys.executable).parent + +#: Executables that would let an agent reach the graph without MCP. A PATH entry +#: holding any of them is dropped; sqlite3 lives in /usr/bin and is left to the hook. +_HIDDEN_EXECUTABLES = ("cgis", "cgis-mcp", "codegraph-brain", "uv", "uvx") +_GRAPH_FILES = ("graph.db", "graph.json") + +ALLOWED_TOOLS = "Read,Grep,Glob,Bash,mcp__cgis" +DISALLOWED_TOOLS = "Write,Edit,NotebookEdit,WebFetch,WebSearch,Task,Agent" +DEFAULT_MODEL = "claude-sonnet-5-5" + + +def _git(*args: str, cwd: Path = _REPO_ROOT) -> str: + """Run a git command, return stdout, raise on failure.""" + result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=True) + return result.stdout.strip() + + +def sanitized_path(path: str) -> str: + """PATH without any directory that holds a cgis or uv executable.""" + kept = [ + d + for d in path.split(os.pathsep) + if d and not any((Path(d) / exe).exists() for exe in _HIDDEN_EXECUTABLES) + ] + return os.pathsep.join(kept) + + +def agent_env(environ: dict[str, str]) -> dict[str, str]: + """The environment a session runs with: sanitized PATH, no active virtualenv.""" + env = {k: v for k, v in environ.items() if k not in ("VIRTUAL_ENV", "PYTHONPATH")} + env["PATH"] = sanitized_path(environ.get("PATH", "")) + return env + + +def remove_graph_files(root: Path) -> list[Path]: + """Delete every graph.db / graph.json under `root` (some repos commit one).""" + found = [p for name in _GRAPH_FILES for p in root.rglob(name) if ".git" not in p.parts] + for path in found: + path.unlink() + return found + + +def mcp_config(arm: Arm) -> dict[str, object]: + """The `--mcp-config` document: empty for control, this checkout's server for cgis.""" + if arm == "control": + return {"mcpServers": {}} + return {"mcpServers": {"cgis": {"command": str(_BIN / "cgis-mcp"), "args": []}}} + + +def hook_settings() -> dict[str, object]: + """The `--settings` document installing the guard hook on every tool call.""" + command = f"{sys.executable} -m cgis.bench.guard" + return { + "hooks": { + "PreToolUse": [{"matcher": ".*", "hooks": [{"type": "command", "command": command}]}] + } + } + + +def stage_plugin(dest: Path) -> Path: + """Copy the plugin without its `.mcp.json`: the server comes from `--mcp-config`. + + The shipped `.mcp.json` launches the PyPI release through uvx, which would + measure a published version rather than this checkout. + """ + shutil.copytree(_REPO_ROOT / "plugin", dest, ignore=shutil.ignore_patterns(".mcp.json")) + return dest + + +def build_command( + *, + claude: str, + prompt: str, + arm: Arm, + config_dir: Path, + model: str, + effort: str | None, + max_budget_usd: float, +) -> list[str]: + """The `claude -p` invocation for one run; writes its config files into `config_dir`.""" + mcp_path = config_dir / "mcp.json" + settings_path = config_dir / "settings.json" + mcp_path.write_text(json.dumps(mcp_config(arm)), encoding="utf-8") + settings_path.write_text(json.dumps(hook_settings()), encoding="utf-8") + cmd = [ + claude, + "-p", + prompt, + "--output-format", + "stream-json", + "--verbose", + "--model", + model, + "--strict-mcp-config", + "--mcp-config", + str(mcp_path), + "--settings", + str(settings_path), + "--setting-sources", + "project", + "--permission-mode", + "dontAsk", + "--allowedTools", + ALLOWED_TOOLS, + "--disallowedTools", + DISALLOWED_TOOLS, + "--no-session-persistence", + "--max-budget-usd", + f"{max_budget_usd:.2f}", + ] + if effort: + cmd += ["--effort", effort] + if arm == "cgis": + cmd += ["--plugin-dir", str(stage_plugin(config_dir / "plugin"))] + return cmd + + +def ingest(worktree: Path, src_root: str) -> float: + """Build `graph.db` in the worktree with this checkout's cgis; seconds taken.""" + start = time.monotonic() + try: + subprocess.run( + [str(_BIN / "cgis"), "ingest", src_root, "--output", "graph.db"], + cwd=worktree, + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as exc: + _msg = f"cgis ingest failed (rc={exc.returncode}):\n{exc.stderr}" + raise RuntimeError(_msg) from exc + return time.monotonic() - start + + +def results_row( + task: AgentTask, + arm: Arm, + run: int, + transcript_lines: Sequence[str], + meta: dict[str, object], +) -> dict[str, object]: + """Score one finished transcript into a results line.""" + transcript = parse_transcript(transcript_lines) + score = score_answer(task, transcript.answer) + answer = extract_answer(transcript.answer) + metrics = run_metrics(transcript, answer.files if answer else []) + return { + "timestamp": datetime.now(UTC).isoformat(), + "task": task.id, + "type": task.type, + "repo": task.repo, + "repo_sha": task.sha, + "arm": arm, + "run": run, + **meta, + "served_model": transcript.model, + "mcp_servers": transcript.mcp_servers, + "subtype": transcript.subtype, + "is_error": transcript.is_error, + "cost_usd": transcript.cost_usd, + "turns": transcript.turns, + "duration_ms": transcript.duration_ms, + "usage": transcript.usage.model_dump(), + "residual_context": transcript.residual_context, + **score.model_dump(), + **metrics.model_dump(), + } + + +def _append_jsonl(path: Path, entry: dict[str, object]) -> None: + """Append one JSON line.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(entry) + "\n") + + +def repo_paths(pairs: Sequence[str]) -> dict[str, Path]: + """`NAME=PATH` arguments as a mapping; this checkout is always `cgis`.""" + repos = {"cgis": _REPO_ROOT} + for pair in pairs: + name, sep, path = pair.partition("=") + if not sep or not name or not path: + _msg = f"--repo expects NAME=PATH, got {pair!r}" + raise ValueError(_msg) + repos[name] = Path(path).expanduser().resolve() + return repos + + +def run_one(task: AgentTask, arm: Arm, run: int, repo: Path, args: argparse.Namespace) -> None: + """One session: worktree → (ingest) → claude -p → transcript → results line.""" + with worktree_at(task.sha, repo) as wt, tempfile.TemporaryDirectory(prefix="ab-") as tmp: + removed = remove_graph_files(wt) + ingest_s = ingest(wt, task.src_root) if arm == "cgis" else 0.0 + cmd = build_command( + claude=args.claude, + prompt=task.prompt(), + arm=arm, + config_dir=Path(tmp), + model=args.model, + effort=args.effort, + max_budget_usd=args.max_budget_usd, + ) + if args.dry_run: + print(f"[{task.id} {arm} #{run}] cwd={wt} removed={len(removed)}") + print(" " + " ".join([*cmd[:2], "", *cmd[3:]])) + return + start = time.monotonic() + proc = subprocess.run( + cmd, + cwd=wt, + env=agent_env(dict(os.environ)), + capture_output=True, + text=True, + timeout=args.timeout, + check=False, + ) + wall_s = time.monotonic() - start + transcript_path = args.transcripts / task.id / f"{arm}-{run}.jsonl" + transcript_path.parent.mkdir(parents=True, exist_ok=True) + transcript_path.write_text(proc.stdout, encoding="utf-8") + meta: dict[str, object] = { + "model": args.model, + "effort": args.effort, + "cgis_sha": _git("rev-parse", "HEAD"), + "returncode": proc.returncode, + "ingest_s": round(ingest_s, 2), + "wall_s": round(wall_s, 2), + "transcript": str(transcript_path.relative_to(_REPO_ROOT)) + if transcript_path.is_relative_to(_REPO_ROOT) + else str(transcript_path), + } + row = results_row(task, arm, run, proc.stdout.splitlines(), meta) + _append_jsonl(args.results, row) + print( + f"[{task.id} {arm} #{run}] recall={row['recall']:.2f} cost=${row['cost_usd']:.3f} " + f"calls={row['total_tool_calls']} reads={row['files_read']} " + f"contaminated={row['contaminated']}" + ) + + +def cmd_run(args: argparse.Namespace) -> int: + """`run`: every selected (task, arm, run) in turn.""" + tasks = load_tasks(args.tasks) + if args.task: + unknown = sorted(set(args.task) - {t.id for t in tasks}) + if unknown: + print(f"unknown task ids: {unknown}", file=sys.stderr) + return 2 + tasks = [t for t in tasks if t.id in args.task] + repos = repo_paths(args.repo) + missing = sorted({t.repo for t in tasks} - repos.keys()) + if missing: + print(f"no --repo NAME=PATH for: {missing}", file=sys.stderr) + return 2 + arms: list[Arm] = args.arm or list(ARMS) + for task in tasks: + for run in range(args.runs): + for arm in arms: + run_one(task, arm, run, repos[task.repo], args) + return 0 + + +def summarize(rows: Sequence[dict[str, object]]) -> list[dict[str, object]]: + """Median per (task, arm) over clean runs; contaminated runs are counted, not scored.""" + groups: dict[tuple[str, str], list[dict[str, object]]] = defaultdict(list) + for row in rows: + groups[(str(row["task"]), str(row["arm"]))].append(row) + summary: list[dict[str, object]] = [] + for (task, arm), group in sorted(groups.items()): + clean = [r for r in group if not r.get("contaminated")] + entry: dict[str, object] = { + "task": task, + "arm": arm, + "runs": len(group), + "contaminated": len(group) - len(clean), + } + for key in ("recall", "precision", "cost_usd", "total_tool_calls", "files_read", "turns"): + values = [float(str(r[key])) for r in clean if r.get(key) is not None] + entry[key] = statistics.median(values) if values else None + summary.append(entry) + return summary + + +def fmt_cell(value: object) -> str: + """A table cell.""" + if value is None: + return "—" + if isinstance(value, float): + return f"{value:.2f}" + return str(value) + + +def cmd_report(args: argparse.Namespace) -> int: + """`report`: a markdown table of medians per (task, arm).""" + if not args.results.exists(): + print(f"no results at {args.results}", file=sys.stderr) + return 1 + lines = args.results.read_text(encoding="utf-8").splitlines() + rows = [json.loads(line) for line in lines if line.strip()] + columns = [ + "task", "arm", "runs", "contaminated", "recall", "precision", + "cost_usd", "total_tool_calls", "files_read", "turns", + ] # fmt: skip + print("| " + " | ".join(columns) + " |") + print("|" + "---|" * len(columns)) + for entry in summarize(rows): + print("| " + " | ".join(fmt_cell(entry[c]) for c in columns) + " |") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + """CLI definition.""" + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + sub = parser.add_subparsers(dest="command", required=True) + + run = sub.add_parser("run", help="run sessions and append results") + run.add_argument("--tasks", type=Path, default=_BENCH_DIR / "tasks") + run.add_argument("--task", action="append", default=[], help="task id; repeatable") + run.add_argument("--arm", action="append", choices=ARMS, help="repeatable; default both") + run.add_argument("--runs", type=int, default=3) + run.add_argument("--repo", action="append", default=[], help="NAME=PATH; repeatable") + run.add_argument("--model", default=DEFAULT_MODEL) + run.add_argument("--effort", default=None) + run.add_argument("--max-budget-usd", type=float, default=2.0) + run.add_argument("--timeout", type=int, default=1800, help="seconds per session") + run.add_argument("--claude", default=shutil.which("claude") or "claude") + run.add_argument("--results", type=Path, default=_BENCH_DIR / "results.jsonl") + run.add_argument("--transcripts", type=Path, default=_BENCH_DIR / "transcripts") + run.add_argument("--dry-run", action="store_true", help="print commands; spend nothing") + run.set_defaults(func=cmd_run) + + report = sub.add_parser("report", help="median per task and arm") + report.add_argument("--results", type=Path, default=_BENCH_DIR / "results.jsonl") + report.set_defaults(func=cmd_report) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Entry point.""" + args = build_parser().parse_args(argv) + code: int = args.func(args) + return code + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/src/cgis/bench/__init__.py b/src/cgis/bench/__init__.py new file mode 100644 index 00000000..cb672d00 --- /dev/null +++ b/src/cgis/bench/__init__.py @@ -0,0 +1 @@ +"""Agent-facing benchmarks: does an agent answer code questions better with cgis (#543).""" diff --git a/src/cgis/bench/agent_task.py b/src/cgis/bench/agent_task.py new file mode 100644 index 00000000..2ce71b34 --- /dev/null +++ b/src/cgis/bench/agent_task.py @@ -0,0 +1,241 @@ +"""Agent A/B tasks, their answer keys, and deterministic answer scoring (#543). + +A task is one code question asked of a headless agent, plus a hand-verified answer +key. The key is written from the code, not from the cgis graph: a key derived from +the graph would agree with every edge the resolver gets wrong, and the benchmark +would score the treatment arm against itself. + +The agent is told to end its answer with a fenced JSON block naming the symbols +and files it relies on. Scoring reads only that block plus a few literal facts, +so it needs no LLM judge and the same transcript always gets the same score. +""" + +import json +import re +from pathlib import Path +from typing import Literal + +import yaml +from pydantic import BaseModel, Field, field_validator + +TaskType = Literal["impact", "flow", "orientation", "audit", "control"] + +#: Appended to every question, identically in both arms. +ANSWER_FORMAT = ( + "\n\nWhen you are done, end your answer with one fenced ```json block of the form " + '{"symbols": [...], "files": [...]}: "symbols" lists the functions, methods or ' + "classes your answer names, as dotted qualified names (module.Class.method); " + '"files" lists the repository-relative paths of the files they live in. ' + "Do not modify any files." +) + +_JSON_BLOCK = re.compile(r"```json\s*\n(.*?)```", re.S) +_CALL_PARENS = re.compile(r"\(.*\)$") +_LINE_SUFFIX = re.compile(r":\d+(?:-\d+)?$") + + +class GoldSymbol(BaseModel, frozen=True): + """One required symbol; any of its spellings counts as found.""" + + id: str + any_of: list[str] = Field(min_length=1) + + +class Gold(BaseModel, frozen=True): + """The answer key for one task. + + `symbols`, `files` and `facts` are required: each one missing costs recall. + `allowed_*` are correct but optional, so naming them costs no precision. + """ + + symbols: list[GoldSymbol] = Field(default_factory=list) + files: list[str] = Field(default_factory=list) + facts: list[str] = Field(default_factory=list) + allowed_symbols: list[str] = Field(default_factory=list) + allowed_files: list[str] = Field(default_factory=list) + + @field_validator("symbols", mode="before") + @classmethod + def _bare_names(cls, value: object) -> object: + """Accept a plain string as a symbol whose only spelling is itself.""" + if not isinstance(value, list): + return value + return [{"id": v, "any_of": [v]} if isinstance(v, str) else v for v in value] + + +class AgentTask(BaseModel, frozen=True): + """One benchmark question pinned to a repository commit.""" + + id: str + repo: str + sha: str + src_root: str + type: TaskType + question: str + gold: Gold + notes: str = "" + + def prompt(self) -> str: + """The exact text sent to the agent.""" + return self.question.strip() + ANSWER_FORMAT + + +class AgentAnswer(BaseModel, frozen=True): + """The structured block parsed from the end of an answer.""" + + symbols: list[str] = Field(default_factory=list) + files: list[str] = Field(default_factory=list) + + +class AnswerScore(BaseModel, frozen=True): + """Recall and precision of one answer against its task's key. + + `recall` pools every required item (symbols, files and facts) so one number + ranks runs; the per-kind fields say where it came from. + """ + + parse_failed: bool + recall: float + precision: float + symbol_recall: float | None + file_recall: float | None + fact_recall: float | None + missed: list[str] + unexpected: list[str] + + +def load_task(path: Path) -> AgentTask: + """Read one task YAML file.""" + return AgentTask.model_validate(yaml.safe_load(path.read_text(encoding="utf-8"))) + + +def load_tasks(directory: Path) -> list[AgentTask]: + """Every `*.yaml` task in a directory, sorted by id; ids must be unique.""" + tasks = sorted((load_task(p) for p in directory.glob("*.yaml")), key=lambda t: t.id) + ids = [t.id for t in tasks] + duplicates = sorted({i for i in ids if ids.count(i) > 1}) + if duplicates: + _msg = f"duplicate task ids: {duplicates}" + raise ValueError(_msg) + return tasks + + +def extract_answer(text: str) -> AgentAnswer | None: + """The last fenced JSON block in `text`, or None when there is none or it is malformed.""" + blocks = _JSON_BLOCK.findall(text) + if not blocks: + return None + try: + data = json.loads(blocks[-1]) + except json.JSONDecodeError: + return None + if not isinstance(data, dict): + return None + try: + return AgentAnswer.model_validate(data) + except ValueError: + return None + + +def normalize_symbol(name: str) -> str: + """Fold the spellings agents use for a symbol into one dotted form. + + `src/cgis/cli.py::structure`, `cgis.cli.structure()` and `cgis/cli.py:structure` + all become a dotted path ending in `cli.structure`. + """ + s = name.strip().strip("`").strip() + s = _CALL_PARENS.sub("", s) + s = s.replace("::", ".").replace(":", ".").replace("/", ".").replace("\\", ".") + s = s.replace(".py.", ".").replace(".ts.", ".").replace(".tsx.", ".") + return s.strip(".") + + +def normalize_file(path: str) -> str: + """Repository-relative POSIX form of a path, without a trailing `:line`.""" + s = path.strip().strip("`").strip().replace("\\", "/") + s = _LINE_SUFFIX.sub("", s) + while s.startswith("./"): + s = s[2:] + return s.lstrip("/") + + +def symbol_matches(predicted: str, gold: str) -> bool: + """True when `predicted` names `gold` on whole dotted components. + + `cgis.cli.structure` names `cli.structure`, and `pkg.ResolverEngine.resolve` + names the class `ResolverEngine`; `cli.structure_x` names neither. + """ + p, g = normalize_symbol(predicted), normalize_symbol(gold) + return bool(g) and f".{g}." in f".{p}." + + +def file_matches(predicted: str, gold: str) -> bool: + """True when two paths name the same file, allowing a missing leading directory. + + The shorter side must still contain a directory, so a bare `cli.py` never + matches: in a real repository a basename alone is ambiguous. + """ + p, g = normalize_file(predicted), normalize_file(gold) + if p == g: + return True + shorter, longer = (p, g) if len(p) < len(g) else (g, p) + return "/" in shorter and longer.endswith("/" + shorter) + + +def _ratio(hit: int, total: int) -> float | None: + """hit/total, or None when there is nothing to count.""" + return hit / total if total else None + + +def _missed_symbols(gold: Gold, answer: AgentAnswer) -> list[str]: + """Ids of required symbols no spelling of which the answer names.""" + return [ + sym.id + for sym in gold.symbols + if not any(symbol_matches(p, g) for p in answer.symbols for g in sym.any_of) + ] + + +def _missed_files(gold: Gold, answer: AgentAnswer) -> list[str]: + """Required files the answer does not name.""" + return [f for f in gold.files if not any(file_matches(p, f) for p in answer.files)] + + +def _missed_facts(gold: Gold, text: str) -> list[str]: + """Required literal facts absent from the answer text (case-insensitive).""" + lowered = text.lower() + return [f"fact:{fact}" for fact in gold.facts if fact.lower() not in lowered] + + +def score_answer(task: AgentTask, text: str) -> AnswerScore: + """Score a final answer against the task's key.""" + gold = task.gold + parsed = extract_answer(text) + answer = parsed or AgentAnswer() + + missed_symbols = _missed_symbols(gold, answer) + missed_files = _missed_files(gold, answer) + missed_facts = _missed_facts(gold, text) + found_symbols = len(gold.symbols) - len(missed_symbols) + found_files = len(gold.files) - len(missed_files) + found_facts = len(gold.facts) - len(missed_facts) + + known_symbols = [g for s in gold.symbols for g in s.any_of] + gold.allowed_symbols + known_files = gold.files + gold.allowed_files + unexpected = [ + p for p in answer.symbols if not any(symbol_matches(p, g) for g in known_symbols) + ] + [p for p in answer.files if not any(file_matches(p, g) for g in known_files)] + predicted = len(answer.symbols) + len(answer.files) + required = len(gold.symbols) + len(gold.files) + len(gold.facts) + found = found_symbols + found_files + found_facts + + return AnswerScore( + parse_failed=parsed is None, + recall=found / required if required else 1.0, + precision=(predicted - len(unexpected)) / predicted if predicted else 0.0, + symbol_recall=_ratio(found_symbols, len(gold.symbols)), + file_recall=_ratio(found_files, len(gold.files)), + fact_recall=_ratio(found_facts, len(gold.facts)), + missed=missed_symbols + missed_files + missed_facts, + unexpected=unexpected, + ) diff --git a/src/cgis/bench/guard.py b/src/cgis/bench/guard.py new file mode 100644 index 00000000..6301d3aa --- /dev/null +++ b/src/cgis/bench/guard.py @@ -0,0 +1,64 @@ +"""PreToolUse hook that keeps cgis out of reach except through MCP (#543). + +Both arms of the agent A/B run with this hook. Without it the control arm is +not a control: codegraph's own benchmark found its control agent invoking their +CLI through Bash in 26 of 28 runs. The treatment arm is blocked too, so it is +measured on the MCP surface alone rather than on whatever mix it improvises. + +Run as `python -m cgis.bench.guard`; Claude Code passes the pending tool call as +JSON on stdin, and exit code 2 refuses it with stderr shown to the agent. The +same predicate is applied to finished transcripts to flag contaminated runs, so +"blocked" and "counted as contamination" cannot drift apart. +""" + +import json +import re +import sys +from collections.abc import Mapping + +#: A cgis-adjacent executable in command position: start of the command, or after +#: a separator, a subshell or a pipe, optionally with a directory in front. +_COMMAND = re.compile( + r"(?:^|[;&|(`]|\$\()\s*(?:\S*/)?(?:cgis|cgis-mcp|codegraph-brain|uvx|uv|sqlite3)\b(?![./-])" +) +_PYTHON_IMPORT = re.compile(r"-m\s+cgis\b|\b(?:import|from)\s+cgis\b") +_GRAPH_FILE = re.compile(r"graph\.(?:db|json)\b") + +_PATH_KEYS = ("file_path", "path", "pattern", "notebook_path") + + +def blocked_reason(tool_name: str, tool_input: Mapping[str, object]) -> str | None: + """Why this tool call must be refused, or None when it may run.""" + if tool_name == "Bash": + command = str(tool_input.get("command", "")) + if _COMMAND.search(command) or _PYTHON_IMPORT.search(command): + return "The cgis CLI, uv and sqlite3 are unavailable in this benchmark." + if _GRAPH_FILE.search(command): + return "The graph database is unavailable in this benchmark." + return None + for key in _PATH_KEYS: + if _GRAPH_FILE.search(str(tool_input.get(key, ""))): + return "The graph database is unavailable in this benchmark." + return None + + +def main() -> int: + """Hook entry point: exit 2 with a reason to refuse the call, 0 to allow it.""" + try: + event = json.load(sys.stdin) + except json.JSONDecodeError: + return 0 + if not isinstance(event, dict): + return 0 + tool_input = event.get("tool_input") + reason = blocked_reason( + str(event.get("tool_name", "")), tool_input if isinstance(tool_input, dict) else {} + ) + if reason is None: + return 0 + print(reason, file=sys.stderr) + return 2 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/src/cgis/bench/transcript.py b/src/cgis/bench/transcript.py new file mode 100644 index 00000000..de8b3038 --- /dev/null +++ b/src/cgis/bench/transcript.py @@ -0,0 +1,284 @@ +"""Read a headless Claude Code transcript (`--output-format stream-json`) into metrics (#543). + +The stream is one JSON event per line: a `system`/`init` event, `assistant` +events carrying `tool_use` blocks and token usage, `user` events carrying the +matching `tool_result` blocks, and one closing `result` event with the answer, +cost and turn count. Everything here is pure, so the metrics are tested on +recorded transcripts without spending anything. + +Two transcript metrics come from codegraph's agent-eval notes and exist to steer +#220, not to rank arms: + +- **sufficiency**: what the agent did right after each cgis answer. Answering + means the answer sufficed; reading a file the answer named means the budget was + spent badly (the source was not included); reading a file it did not name means + the answer missed something; searching means it was not trusted. +- **allocation**: of the files cgis answers named, the share the final answer + relied on. Measured per file, not per byte: cgis output does not delimit bytes + by file, and a byte split would be an invention. +""" + +import json +import re +from collections import Counter +from collections.abc import Iterable +from typing import Literal + +from pydantic import BaseModel, Field + +from cgis.bench.agent_task import file_matches +from cgis.bench.guard import blocked_reason + +CGIS_TOOL_PREFIX = "mcp__cgis__" + +Sufficiency = Literal[ + "answered", "called_again", "read_returned", "read_other", "searched", "other" +] + +_SEARCH_TOOLS = frozenset({"Grep", "Glob", "Bash"}) +_SOURCE_PATH = re.compile(r"[\w.\-/]+\.(?:py|tsx?|jsx?)\b") + + +class ToolCall(BaseModel, frozen=True): + """One tool invocation and its result, in transcript order.""" + + id: str + name: str + input: dict[str, object] + result: str = "" + is_error: bool = False + #: The Task/Agent call this ran under, or None on the main thread. + parent: str | None = None + + +class Usage(BaseModel, frozen=True): + """Token counts as the API reports them.""" + + input_tokens: int = 0 + output_tokens: int = 0 + cache_creation_input_tokens: int = 0 + cache_read_input_tokens: int = 0 + + @property + def context(self) -> int: + """Tokens the request carried in its context window.""" + return self.input_tokens + self.cache_creation_input_tokens + self.cache_read_input_tokens + + +class Transcript(BaseModel, frozen=True): + """A parsed run.""" + + cwd: str = "" + model: str = "" + mcp_servers: dict[str, str] = Field(default_factory=dict) + calls: list[ToolCall] = Field(default_factory=list) + answer: str = "" + subtype: str = "" + is_error: bool = False + cost_usd: float = 0.0 + turns: int = 0 + duration_ms: int = 0 + usage: Usage = Field(default_factory=Usage) + #: Context size of the last main-thread request: what the session ends holding. + residual_context: int = 0 + + +class RunMetrics(BaseModel, frozen=True): + """Everything one results row records about agent behaviour.""" + + tool_calls: dict[str, int] + total_tool_calls: int + cgis_calls: int + files_read: int + bytes_read: int + cli_attempts: int + contaminated: bool + sufficiency: dict[str, int] + allocation: float | None + + +def _result_text(content: object) -> str: + """Flatten a tool_result `content` (a string or a list of blocks) to text.""" + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join(str(block.get("text", "")) for block in content if isinstance(block, dict)) + return "" + + +def _blocks(event: dict[str, object]) -> list[dict[str, object]]: + """The content blocks of an assistant or user event.""" + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None + return [b for b in content if isinstance(b, dict)] if isinstance(content, list) else [] + + +def _usage(raw: object) -> Usage: + """A Usage from an API usage dict, ignoring fields it does not model.""" + if not isinstance(raw, dict): + return Usage() + return Usage.model_validate({k: v for k, v in raw.items() if k in Usage.model_fields}) + + +class _Builder: + """Accumulates events; split out so `parse_transcript` stays a plain loop.""" + + def __init__(self) -> None: + self.fields: dict[str, object] = {} + self.calls: dict[str, ToolCall] = {} + self.residual = 0 + + def init(self, event: dict[str, object]) -> None: + """Record the session's working directory, model and MCP server states.""" + self.fields["cwd"] = str(event.get("cwd", "")) + self.fields["model"] = str(event.get("model", "")) + servers = event.get("mcp_servers") + if isinstance(servers, list): + self.fields["mcp_servers"] = { + str(s.get("name")): str(s.get("status")) for s in servers if isinstance(s, dict) + } + + def assistant(self, event: dict[str, object]) -> None: + """Record tool calls, and the context size of main-thread requests.""" + parent = event.get("parent_tool_use_id") + parent_id = parent if isinstance(parent, str) else None + for block in _blocks(event): + if block.get("type") == "tool_use": + call_id = str(block.get("id")) + raw_input = block.get("input") + self.calls[call_id] = ToolCall( + id=call_id, + name=str(block.get("name")), + input=raw_input if isinstance(raw_input, dict) else {}, + parent=parent_id, + ) + message = event.get("message") + if parent_id is None and isinstance(message, dict): + self.residual = _usage(message.get("usage")).context + + def user(self, event: dict[str, object]) -> None: + """Attach each tool_result to the call it answers.""" + for block in _blocks(event): + call = self.calls.get(str(block.get("tool_use_id"))) + if block.get("type") == "tool_result" and call is not None: + self.calls[call.id] = call.model_copy( + update={ + "result": _result_text(block.get("content")), + "is_error": bool(block.get("is_error", False)), + } + ) + + def result(self, event: dict[str, object]) -> None: + """Record the final answer, cost and totals.""" + self.fields.update( + answer=str(event.get("result", "")), + subtype=str(event.get("subtype", "")), + is_error=bool(event.get("is_error", False)), + cost_usd=float(str(event.get("total_cost_usd", 0) or 0)), + turns=int(str(event.get("num_turns", 0) or 0)), + duration_ms=int(str(event.get("duration_ms", 0) or 0)), + usage=_usage(event.get("usage")), + ) + + +def parse_transcript(lines: Iterable[str]) -> Transcript: + """Parse stream-json lines; blank and non-JSON lines are skipped.""" + builder = _Builder() + handlers = { + "system": builder.init, + "assistant": builder.assistant, + "user": builder.user, + "result": builder.result, + } + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict): + continue + handler = handlers.get(str(event.get("type"))) + if handler is not None and ( + event.get("type") != "system" or event.get("subtype") == "init" + ): + handler(event) + return Transcript.model_validate( + { + **builder.fields, + "calls": list(builder.calls.values()), + "residual_context": builder.residual, + } + ) + + +def is_cgis_call(call: ToolCall) -> bool: + """True for a call to a cgis MCP tool.""" + return call.name.startswith(CGIS_TOOL_PREFIX) + + +def _relative(path: str, cwd: str) -> str: + """`path` relative to the session's working directory when it lies inside it.""" + prefix = cwd.rstrip("/") + "/" + return path[len(prefix) :] if cwd and path.startswith(prefix) else path + + +def returned_files(call: ToolCall) -> set[str]: + """Source paths a tool result names.""" + return set(_SOURCE_PATH.findall(call.result)) + + +def classify_next(returned: set[str], following: ToolCall | None, cwd: str) -> Sufficiency: + """What the agent did right after a cgis answer naming `returned` files.""" + if following is None: + return "answered" + if is_cgis_call(following): + return "called_again" + if following.name == "Read": + path = _relative(str(following.input.get("file_path", "")), cwd) + hit = any(file_matches(path, f) for f in returned) + return "read_returned" if hit else "read_other" + if following.name in _SEARCH_TOOLS: + return "searched" + return "other" + + +def sufficiency(transcript: Transcript) -> Counter[str]: + """Count of `classify_next` outcomes over every main-thread cgis call.""" + main = [c for c in transcript.calls if c.parent is None] + counts: Counter[str] = Counter() + for i, call in enumerate(main): + if is_cgis_call(call): + following = main[i + 1] if i + 1 < len(main) else None + counts[classify_next(returned_files(call), following, transcript.cwd)] += 1 + return counts + + +def allocation(transcript: Transcript, answer_files: list[str]) -> float | None: + """Share of files named by cgis answers that the final answer relied on, pooled.""" + named = 0 + used = 0 + for call in transcript.calls: + if not is_cgis_call(call): + continue + files = returned_files(call) + named += len(files) + used += sum(1 for f in files if any(file_matches(a, f) for a in answer_files)) + return used / named if named else None + + +def run_metrics(transcript: Transcript, answer_files: list[str]) -> RunMetrics: + """Every behavioural metric for one run.""" + counts = Counter(c.name for c in transcript.calls) + reads = [c for c in transcript.calls if c.name == "Read" and not c.is_error] + attempts = [c for c in transcript.calls if blocked_reason(c.name, c.input) is not None] + return RunMetrics( + tool_calls=dict(sorted(counts.items())), + total_tool_calls=len(transcript.calls), + cgis_calls=sum(1 for c in transcript.calls if is_cgis_call(c)), + files_read=len({str(c.input.get("file_path", "")) for c in reads}), + bytes_read=sum(len(c.result.encode("utf-8")) for c in reads), + cli_attempts=len(attempts), + contaminated=any(not c.is_error for c in attempts), + sufficiency=dict(sorted(sufficiency(transcript).items())), + allocation=allocation(transcript, answer_files), + ) diff --git a/tests/self_parsing/test_receiver_resolution.py b/tests/self_parsing/test_receiver_resolution.py index d813ac77..07567369 100644 --- a/tests/self_parsing/test_receiver_resolution.py +++ b/tests/self_parsing/test_receiver_resolution.py @@ -38,7 +38,12 @@ #: has no node to resolve to. The centre was 46 and had drifted 8 low, so the band #: was measuring less each time something was added to a store method rather than #: catching a regression. -_EXPECTED_PLACEHOLDERS = 57 +#: +#: Re-measured 2026-10-02: 66 on `main` (715cd9a), 69 with `cgis.bench` (#543). The +#: three new ones are `self.question.strip`, `self.calls.get` and `self.fields.update`: +#: `str` and `dict` receivers, declined by D1 like the ones above. `main` alone sat +#: 9 above the old centre, one short of the edge. +_EXPECTED_PLACEHOLDERS = 69 _TOLERANCE = 10 diff --git a/tests/unit/agent_ab_stubs.py b/tests/unit/agent_ab_stubs.py new file mode 100644 index 00000000..c0bf4a8f --- /dev/null +++ b/tests/unit/agent_ab_stubs.py @@ -0,0 +1,110 @@ +"""Builders for synthetic stream-json transcripts in the shape `claude -p` emits (#543).""" + +import json + + +def init(cwd: str = "/w", servers: dict[str, str] | None = None) -> dict[str, object]: + """The session's opening `system`/`init` event.""" + return { + "type": "system", + "subtype": "init", + "cwd": cwd, + "model": "claude-sonnet-5-5", + "mcp_servers": [{"name": k, "status": v} for k, v in (servers or {}).items()], + } + + +def tool_use( + call_id: str, + name: str, + tool_input: dict[str, object], + *, + context: int = 1000, + parent: str | None = None, +) -> dict[str, object]: + """An assistant event issuing one tool call; `context` is its cache-read size.""" + return { + "type": "assistant", + "parent_tool_use_id": parent, + "message": { + "content": [{"type": "tool_use", "id": call_id, "name": name, "input": tool_input}], + "usage": {"input_tokens": 10, "cache_read_input_tokens": context, "output_tokens": 5}, + }, + } + + +def tool_result(call_id: str, content: object, *, is_error: bool = False) -> dict[str, object]: + """The user event carrying a tool's result.""" + return { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": call_id, + "content": content, + "is_error": is_error, + } + ] + }, + } + + +def result(answer: str, *, cost: float = 0.25, turns: int = 6) -> dict[str, object]: + """The closing `result` event.""" + return { + "type": "result", + "subtype": "success", + "is_error": False, + "duration_ms": 41000, + "num_turns": turns, + "result": answer, + "total_cost_usd": cost, + "usage": { + "input_tokens": 120, + "output_tokens": 900, + "cache_creation_input_tokens": 30000, + "cache_read_input_tokens": 400000, + }, + } + + +def answer_text(symbols: list[str], files: list[str]) -> str: + """A final answer ending in the required JSON block.""" + block = json.dumps({"symbols": symbols, "files": files}) + return f"Here is what I found.\n\n```json\n{block}\n```" + + +def lines(*events: dict[str, object]) -> list[str]: + """Events serialised one per line, as the CLI writes them.""" + return [json.dumps(e) for e in events] + + +def cgis_session() -> list[str]: + """A treatment-arm session: context, a read of a returned file, a grep, an answer.""" + return lines( + init(servers={"cgis": "connected"}), + tool_use("c1", "mcp__cgis__cgis_analyze_impact", {"fqn": "language_for"}, context=2000), + tool_result( + "c1", + [ + { + "type": "text", + "text": "callers: cgis/cli.py structure; cgis/guardian/collector.py", + } + ], + ), + tool_use("r1", "Read", {"file_path": "/w/src/cgis/cli.py"}, context=5000), + tool_result("r1", "def structure(): ...\n"), + tool_use("c2", "mcp__cgis__cgis_context", {"fqn": "x"}, context=6000), + tool_result("c2", "see cgis/query/render/mermaid.py"), + tool_use("g1", "Grep", {"pattern": "language_for", "path": "/w/src"}, context=7000), + tool_result("g1", "src/cgis/extractors/registry.py:88"), + tool_use("s1", "Read", {"file_path": "/w/x.py"}, context=9999, parent="task-1"), + result( + answer_text( + ["cgis.cli.structure", "cgis.extractors.registry.is_supported"], + ["src/cgis/cli.py", "src/cgis/extractors/registry.py"], + ) + ), + ) diff --git a/tests/unit/test_agent_ab_guard.py b/tests/unit/test_agent_ab_guard.py new file mode 100644 index 00000000..3056c280 --- /dev/null +++ b/tests/unit/test_agent_ab_guard.py @@ -0,0 +1,88 @@ +"""The PreToolUse guard that keeps cgis out of both A/B arms except via MCP (#543).""" + +import io +import json + +import pytest + +from cgis.bench import guard +from cgis.bench.guard import blocked_reason + + +@pytest.mark.parametrize( + "command", + [ + "cgis impact cgis.cli.ingest", + "cd src && cgis find language_for", + "/home/u/.venv/bin/cgis trace x", + "uv run cgis overview", + "uvx codegraph-brain", + "echo $(cgis-mcp)", + "sqlite3 graph.db 'select 1'", + "python3 -m cgis overview", + 'python3 -c "from cgis.cli import app"', + "ls -la graph.db", + "cat ui/public/graph.json | head", + "find . | xargs grep x; uv pip list", + ], +) +def test_bash_routes_to_the_graph_are_blocked(command: str) -> None: + assert blocked_reason("Bash", {"command": command}) is not None + + +@pytest.mark.parametrize( + "command", + [ + "grep -rn language_for src/cgis", + "ls src/cgis/extractors", + "cat uv.lock | head", + "git log --oneline -3", + "rg 'def ingest' src/cgis/cli.py", + "python3 -c 'print(1)'", + ], +) +def test_ordinary_exploration_is_allowed(command: str) -> None: + """This repository's own paths contain `cgis`; only command position counts.""" + assert blocked_reason("Bash", {"command": command}) is None + + +@pytest.mark.parametrize( + ("tool", "tool_input"), + [ + ("Read", {"file_path": "/w/graph.db"}), + ("Read", {"file_path": "/w/ui/public/graph.json"}), + ("Grep", {"pattern": "x", "path": "graph.db-wal"}), + ("Glob", {"pattern": "**/graph.json"}), + ], +) +def test_file_tools_cannot_open_graph_files(tool: str, tool_input: dict[str, object]) -> None: + assert blocked_reason(tool, tool_input) is not None + + +def test_file_tools_on_source_are_allowed() -> None: + assert blocked_reason("Read", {"file_path": "/w/src/cgis/cli.py"}) is None + assert blocked_reason("mcp__cgis__cgis_context", {"fqn": "cgis.cli.ingest"}) is None + + +def _run_main(monkeypatch: pytest.MonkeyPatch, stdin: str) -> int: + monkeypatch.setattr("sys.stdin", io.StringIO(stdin)) + return guard.main() + + +def test_main_refuses_with_exit_2_and_a_reason( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + event = {"tool_name": "Bash", "tool_input": {"command": "cgis overview"}} + assert _run_main(monkeypatch, json.dumps(event)) == 2 + assert "unavailable" in capsys.readouterr().err + + +def test_main_allows_ordinary_calls(monkeypatch: pytest.MonkeyPatch) -> None: + event = {"tool_name": "Bash", "tool_input": {"command": "ls"}} + assert _run_main(monkeypatch, json.dumps(event)) == 0 + + +@pytest.mark.parametrize("stdin", ["not json", "[1]", '{"tool_name": "Bash", "tool_input": 3}']) +def test_main_lets_malformed_events_through(monkeypatch: pytest.MonkeyPatch, stdin: str) -> None: + """Fail open: a malformed event is still caught afterwards as contamination.""" + assert _run_main(monkeypatch, stdin) == 0 diff --git a/tests/unit/test_agent_ab_script.py b/tests/unit/test_agent_ab_script.py new file mode 100644 index 00000000..9cbd5f55 --- /dev/null +++ b/tests/unit/test_agent_ab_script.py @@ -0,0 +1,251 @@ +"""The agent A/B runner `scripts/agent_ab.py` (#543), without spending a session.""" + +import argparse +import json +import subprocess +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import agent_ab_stubs as stub +import pytest + +# This repository sets no pytest `pythonpath`; scripts are imported explicitly. +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "scripts")) + +import agent_ab as ab + +from cgis.bench.agent_task import load_task + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +TASK = load_task(REPO_ROOT / "benchmarks" / "agent_ab" / "tasks" / "cgis-impact-language-for.yaml") + + +def _args(tmp_path: Path, **overrides: object) -> argparse.Namespace: + args = ab.build_parser().parse_args(["run"]) + args.results = tmp_path / "results.jsonl" + args.transcripts = tmp_path / "transcripts" + args.claude = "claude" + for key, value in overrides.items(): + setattr(args, key, value) + return args + + +# --- environment ------------------------------------------------------------ + + +def test_sanitized_path_drops_dirs_holding_cgis_or_uv(tmp_path: Path) -> None: + clean, venv, uvdir = tmp_path / "clean", tmp_path / "venv", tmp_path / "uv" + for d in (clean, venv, uvdir): + d.mkdir() + (venv / "cgis").touch() + (uvdir / "uvx").touch() + joined = ":".join([str(clean), str(venv), "", str(uvdir)]) + assert ab.sanitized_path(joined) == str(clean) + + +def test_agent_env_drops_the_virtualenv(tmp_path: Path) -> None: + env = ab.agent_env( + {"PATH": str(tmp_path), "VIRTUAL_ENV": "/v", "PYTHONPATH": "/p", "HOME": "/h"} + ) + assert env == {"PATH": str(tmp_path), "HOME": "/h"} + + +def test_remove_graph_files_finds_nested_and_committed_copies(tmp_path: Path) -> None: + (tmp_path / "ui" / "public").mkdir(parents=True) + (tmp_path / ".git").mkdir() + (tmp_path / "graph.db").touch() + (tmp_path / "ui" / "public" / "graph.json").touch() + (tmp_path / ".git" / "graph.db").touch() + removed = ab.remove_graph_files(tmp_path) + assert sorted(p.relative_to(tmp_path).as_posix() for p in removed) == [ + "graph.db", + "ui/public/graph.json", + ] + assert (tmp_path / ".git" / "graph.db").exists() + + +# --- command ------------------------------------------------------------------ + + +def _command(tmp_path: Path, arm: ab.Arm, effort: str | None = None) -> list[str]: + return ab.build_command( + claude="claude", + prompt="Q?", + arm=arm, + config_dir=tmp_path, + model="m", + effort=effort, + max_budget_usd=1.5, + ) + + +def test_control_command_has_no_server_and_no_plugin(tmp_path: Path) -> None: + cmd = _command(tmp_path, "control") + assert cmd[:3] == ["claude", "-p", "Q?"] + assert "--plugin-dir" not in cmd + assert "--effort" not in cmd + assert cmd[cmd.index("--max-budget-usd") + 1] == "1.50" + assert "--strict-mcp-config" in cmd + assert json.loads((tmp_path / "mcp.json").read_text()) == {"mcpServers": {}} + + +def test_both_arms_install_the_guard_hook(tmp_path: Path) -> None: + _command(tmp_path, "control") + settings = json.loads((tmp_path / "settings.json").read_text()) + hook = settings["hooks"]["PreToolUse"][0] + assert hook["matcher"] == ".*" + assert hook["hooks"][0]["command"].endswith("-m cgis.bench.guard") + + +def test_cgis_command_uses_this_checkouts_server_and_the_plugin_without_mcp_json( + tmp_path: Path, +) -> None: + cmd = _command(tmp_path, "cgis", effort="high") + server = json.loads((tmp_path / "mcp.json").read_text())["mcpServers"]["cgis"] + assert server["command"].endswith("cgis-mcp") + plugin = Path(cmd[cmd.index("--plugin-dir") + 1]) + assert (plugin / "skills" / "cgis" / "SKILL.md").is_file() + assert not (plugin / ".mcp.json").exists() + assert cmd[cmd.index("--effort") + 1] == "high" + + +def test_repo_paths_parses_name_path_pairs(tmp_path: Path) -> None: + repos = ab.repo_paths([f"owner-api={tmp_path}"]) + assert repos["owner-api"] == tmp_path.resolve() + assert repos["cgis"] == REPO_ROOT + + +@pytest.mark.parametrize("pair", ["no-equals", "=path", "name="]) +def test_repo_paths_rejects_malformed_pairs(pair: str) -> None: + with pytest.raises(ValueError, match="NAME=PATH"): + ab.repo_paths([pair]) + + +def test_ingest_reports_failures_with_stderr( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + def fail(*_a: object, **_k: object) -> None: + raise subprocess.CalledProcessError(1, "cgis", stderr="boom") + + monkeypatch.setattr(ab.subprocess, "run", fail) + with pytest.raises(RuntimeError, match="boom"): + ab.ingest(tmp_path, "src") + + +# --- running ------------------------------------------------------------------ + + +@contextmanager +def _fake_worktree(path: Path) -> Iterator[Path]: + path.mkdir(exist_ok=True) + (path / "graph.json").touch() + yield path + + +def _patch_run(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> list[list[str]]: + calls: list[list[str]] = [] + + def fake_run(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]: + calls.append(cmd) + if cmd[0] == "git": + return subprocess.CompletedProcess(cmd, 0, stdout="cafe\n", stderr="") + out = "\n".join(stub.cgis_session()) + return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="") + + monkeypatch.setattr(ab, "worktree_at", lambda _sha, _repo: _fake_worktree(tmp_path / "wt")) + monkeypatch.setattr(ab, "ingest", lambda _wt, _src: 1.25) + monkeypatch.setattr(ab.subprocess, "run", fake_run) + return calls + + +def test_dry_run_prints_commands_and_spends_nothing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + calls = _patch_run(monkeypatch, tmp_path) + args = _args(tmp_path, dry_run=True, runs=1, task=[TASK.id]) + assert ab.cmd_run(args) == 0 + out = capsys.readouterr().out + assert f"[{TASK.id} control #0]" in out + assert f"[{TASK.id} cgis #0]" in out + assert "removed=1" in out + assert calls == [] + assert not args.results.exists() + + +def test_a_run_writes_its_transcript_and_a_scored_results_line( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _patch_run(monkeypatch, tmp_path) + args = _args(tmp_path, runs=1, task=[TASK.id], arm=["cgis"]) + assert ab.cmd_run(args) == 0 + rows = [json.loads(line) for line in args.results.read_text().splitlines()] + assert len(rows) == 1 + row = rows[0] + assert (row["task"], row["arm"], row["run"]) == (TASK.id, "cgis", 0) + assert row["cgis_sha"] == "cafe" + assert row["ingest_s"] == 1.25 + assert row["cost_usd"] == 0.25 + assert row["recall"] == pytest.approx(4 / 9) + assert row["cgis_calls"] == 2 + assert row["contaminated"] is False + assert (args.transcripts / TASK.id / "cgis-0.jsonl").is_file() + + +def test_unknown_task_ids_and_missing_repos_are_refused(tmp_path: Path) -> None: + assert ab.cmd_run(_args(tmp_path, task=["nope"])) == 2 + tasks = tmp_path / "tasks" + tasks.mkdir() + text = (REPO_ROOT / "benchmarks/agent_ab/tasks/cgis-impact-language-for.yaml").read_text() + (tasks / "t.yaml").write_text(text.replace("repo: cgis", "repo: owner-api")) + assert ab.cmd_run(_args(tmp_path, tasks=tasks)) == 2 + + +# --- reporting ---------------------------------------------------------------- + + +def _row(task: str, arm: str, recall: float, *, contaminated: bool = False) -> dict[str, object]: + return { + "task": task, + "arm": arm, + "recall": recall, + "precision": 1.0, + "cost_usd": 0.3, + "total_tool_calls": 10, + "files_read": 4, + "turns": 8, + "contaminated": contaminated, + } + + +def test_summarize_takes_medians_over_clean_runs_only() -> None: + rows = [ + _row("t", "cgis", 1.0), + _row("t", "cgis", 0.5), + _row("t", "cgis", 0.0, contaminated=True), + _row("t", "control", 0.2, contaminated=True), + ] + summary = {(s["task"], s["arm"]): s for s in ab.summarize(rows)} + assert summary[("t", "cgis")]["recall"] == 0.75 + assert summary[("t", "cgis")]["runs"] == 3 + assert summary[("t", "cgis")]["contaminated"] == 1 + assert summary[("t", "control")]["recall"] is None + + +def test_report_prints_a_markdown_table(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + results = tmp_path / "results.jsonl" + results.write_text(json.dumps(_row("t", "cgis", 1.0)) + "\n\n") + assert ab.main(["report", "--results", str(results)]) == 0 + out = capsys.readouterr().out.splitlines() + assert out[0].startswith("| task | arm |") + assert out[2] == "| t | cgis | 1 | 0 | 1.00 | 1.00 | 0.30 | 10.00 | 4.00 | 8.00 |" + + +def test_report_without_results_fails(tmp_path: Path) -> None: + assert ab.main(["report", "--results", str(tmp_path / "none.jsonl")]) == 1 + + +def test_fmt_renders_missing_values_as_a_dash() -> None: + assert ab.fmt_cell(None) == "—" + assert ab.fmt_cell(3) == "3" diff --git a/tests/unit/test_agent_ab_task.py b/tests/unit/test_agent_ab_task.py new file mode 100644 index 00000000..b5a9537f --- /dev/null +++ b/tests/unit/test_agent_ab_task.py @@ -0,0 +1,206 @@ +"""Agent A/B tasks and answer scoring (#543).""" + +import json +from pathlib import Path + +import pytest + +from cgis.bench.agent_task import ( + ANSWER_FORMAT, + AgentTask, + extract_answer, + file_matches, + load_task, + load_tasks, + normalize_file, + normalize_symbol, + score_answer, + symbol_matches, +) + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +TASKS_DIR = REPO_ROOT / "benchmarks" / "agent_ab" / "tasks" + + +def _task(**gold: object) -> AgentTask: + return AgentTask.model_validate( + { + "id": "t", + "repo": "cgis", + "sha": "abc", + "src_root": "src", + "type": "impact", + "question": "Who calls f?", + "gold": gold, + } + ) + + +def _answer(symbols: list[str], files: list[str], prose: str = "") -> str: + return f"{prose}\n\n```json\n{json.dumps({'symbols': symbols, 'files': files})}\n```\n" + + +# --- the shipped tasks ------------------------------------------------------ + + +def test_shipped_tasks_load_with_unique_ids() -> None: + tasks = load_tasks(TASKS_DIR) + assert len(tasks) >= 4 + assert len({t.id for t in tasks}) == len(tasks) + + +def test_shipped_cgis_keys_name_files_that_exist() -> None: + """A key naming a moved file can never be met, so every run would score a miss.""" + for task in load_tasks(TASKS_DIR): + if task.repo != "cgis": + continue + for path in task.gold.files + task.gold.allowed_files: + assert (REPO_ROOT / path).is_file(), f"{task.id}: {path}" + + +def test_duplicate_ids_are_refused(tmp_path: Path) -> None: + text = (TASKS_DIR / "cgis-impact-language-for.yaml").read_text(encoding="utf-8") + (tmp_path / "a.yaml").write_text(text, encoding="utf-8") + (tmp_path / "b.yaml").write_text(text, encoding="utf-8") + with pytest.raises(ValueError, match="duplicate task ids"): + load_tasks(tmp_path) + + +def test_prompt_appends_the_answer_format() -> None: + task = load_task(TASKS_DIR / "cgis-impact-language-for.yaml") + assert task.prompt().endswith(ANSWER_FORMAT) + assert task.prompt().startswith("In this repository") + + +def test_bare_gold_symbols_become_single_spelling_entries() -> None: + task = _task(symbols=["a.b", {"id": "x", "any_of": ["X.y", "Z.y"]}]) + assert [s.id for s in task.gold.symbols] == ["a.b", "x"] + assert task.gold.symbols[0].any_of == ["a.b"] + + +# --- answer extraction ------------------------------------------------------ + + +def test_extract_answer_takes_the_last_json_block() -> None: + text = _answer(["first"], []) + _answer(["second"], ["f.py"]) + answer = extract_answer(text) + assert answer is not None + assert answer.symbols == ["second"] + assert answer.files == ["f.py"] + + +@pytest.mark.parametrize( + "text", + [ + "no block at all", + "```json\n{not json}\n```", + "```json\n[1, 2]\n```", + '```json\n{"symbols": "not-a-list"}\n```', + ], +) +def test_extract_answer_rejects_missing_or_malformed_blocks(text: str) -> None: + assert extract_answer(text) is None + + +# --- normalisation and matching -------------------------------------------- + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("src/cgis/cli.py::structure", "src.cgis.cli.structure"), + ("`cgis.cli.structure()`", "cgis.cli.structure"), + ("cgis/cli.py:structure", "cgis.cli.structure"), + ("ui/src/a.tsx::render", "ui.src.a.render"), + ], +) +def test_normalize_symbol(raw: str, expected: str) -> None: + assert normalize_symbol(raw) == expected + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("./src/cgis/cli.py", "src/cgis/cli.py"), + ("src/cgis/cli.py:843", "src/cgis/cli.py"), + ("src\\cgis\\cli.py:10-20", "src/cgis/cli.py"), + ("/src/cgis/cli.py", "src/cgis/cli.py"), + ], +) +def test_normalize_file(raw: str, expected: str) -> None: + assert normalize_file(raw) == expected + + +@pytest.mark.parametrize( + ("predicted", "gold", "expected"), + [ + ("cgis.cli.structure", "cli.structure", True), + ("src/cgis/cli.py::structure", "cli.structure", True), + ("pkg.ResolverEngine.resolve", "ResolverEngine", True), + ("cgis.cli.structure_x", "cli.structure", False), + ("structure", "cli.structure", False), + ("", "", False), + ], +) +def test_symbol_matches(predicted: str, gold: str, expected: bool) -> None: + assert symbol_matches(predicted, gold) is expected + + +@pytest.mark.parametrize( + ("predicted", "gold", "expected"), + [ + ("src/cgis/cli.py", "src/cgis/cli.py", True), + ("cgis/cli.py", "src/cgis/cli.py", True), + ("/abs/root/src/cgis/cli.py", "src/cgis/cli.py", True), + ("cli.py", "src/cgis/cli.py", False), + ("src/cgis/xcli.py", "src/cgis/cli.py", False), + ], +) +def test_file_matches(predicted: str, gold: str, expected: bool) -> None: + assert file_matches(predicted, gold) is expected + + +# --- scoring ---------------------------------------------------------------- + + +def test_perfect_answer_scores_one() -> None: + task = _task(symbols=["mod.f", "mod.g"], files=["src/mod.py"], facts=["WAL"]) + score = score_answer(task, _answer(["pkg.mod.f", "pkg.mod.g"], ["src/mod.py"], "uses WAL")) + assert score.parse_failed is False + assert score.recall == 1.0 + assert score.precision == 1.0 + assert (score.symbol_recall, score.file_recall, score.fact_recall) == (1.0, 1.0, 1.0) + assert score.missed == [] + assert score.unexpected == [] + + +def test_partial_answer_reports_what_was_missed_and_what_was_extra() -> None: + task = _task( + symbols=["mod.f", {"id": "parse", "any_of": ["A.parse", "B.parse"]}], + files=["src/mod.py", "src/other.py"], + facts=["5000"], + allowed_symbols=["mod.helper"], + ) + text = _answer(["mod.f", "B.parse", "mod.helper", "mod.wrong"], ["src/mod.py"]) + score = score_answer(task, text) + assert score.symbol_recall == 1.0 + assert score.file_recall == 0.5 + assert score.fact_recall == 0.0 + assert score.recall == pytest.approx(3 / 5) + assert score.missed == ["src/other.py", "fact:5000"] + assert score.unexpected == ["mod.wrong"] + assert score.precision == pytest.approx(4 / 5) + + +def test_unparseable_answer_is_flagged_and_scores_zero_precision() -> None: + task = _task(symbols=["mod.f"]) + score = score_answer(task, "It is called from mod.f.") + assert score.parse_failed is True + assert score.recall == 0.0 + assert score.precision == 0.0 + assert score.file_recall is None + + +def test_task_without_required_items_has_full_recall() -> None: + score = score_answer(_task(), _answer([], [])) + assert score.recall == 1.0 diff --git a/tests/unit/test_agent_ab_transcript.py b/tests/unit/test_agent_ab_transcript.py new file mode 100644 index 00000000..1e8a7775 --- /dev/null +++ b/tests/unit/test_agent_ab_transcript.py @@ -0,0 +1,125 @@ +"""Parsing headless Claude Code transcripts into A/B metrics (#543).""" + +import agent_ab_stubs as stub + +from cgis.bench.transcript import ( + ToolCall, + Transcript, + allocation, + classify_next, + parse_transcript, + returned_files, + run_metrics, + sufficiency, +) + + +def test_parse_reads_session_header_calls_and_closing_totals() -> None: + t = parse_transcript(stub.cgis_session()) + assert t.cwd == "/w" + assert t.model == "claude-sonnet-5-5" + assert t.mcp_servers == {"cgis": "connected"} + assert [c.id for c in t.calls] == ["c1", "r1", "c2", "g1", "s1"] + assert t.calls[0].result.startswith("callers:") + assert t.calls[4].parent == "task-1" + assert t.cost_usd == 0.25 + assert t.turns == 6 + assert t.duration_ms == 41000 + assert t.usage.cache_read_input_tokens == 400000 + assert t.subtype == "success" + assert "```json" in t.answer + + +def test_residual_context_is_the_last_main_thread_request() -> None: + """A sub-agent's request does not describe the main session's context.""" + t = parse_transcript(stub.cgis_session()) + assert t.residual_context == 10 + 7000 + + +def test_parse_skips_noise_lines_and_non_init_system_events() -> None: + t = parse_transcript( + ["", "not json", "[1]", *stub.lines({"type": "system", "subtype": "hook", "cwd": "/x"})] + ) + assert t == Transcript() + + +def test_tool_result_for_an_unknown_call_is_ignored() -> None: + t = parse_transcript(stub.lines(stub.tool_result("nope", "x"))) + assert t.calls == [] + + +def test_error_results_are_kept_as_errors() -> None: + t = parse_transcript( + stub.lines( + stub.tool_use("b", "Bash", {"command": "cgis x"}), + stub.tool_result("b", "no", is_error=True), + ) + ) + assert t.calls[0].is_error is True + + +def test_returned_files_finds_source_paths() -> None: + call = ToolCall(id="c", name="mcp__cgis__x", input={}, result="a/b.py, c.ts and d.tsx; e.md") + assert returned_files(call) == {"a/b.py", "c.ts", "d.tsx"} + + +def _call(name: str, **tool_input: object) -> ToolCall: + return ToolCall(id="n", name=name, input=dict(tool_input)) + + +def test_classify_next_outcomes() -> None: + returned = {"cgis/cli.py"} + assert classify_next(returned, None, "/w") == "answered" + assert classify_next(returned, _call("mcp__cgis__cgis_trace_flow"), "/w") == "called_again" + assert ( + classify_next(returned, _call("Read", file_path="/w/src/cgis/cli.py"), "/w") + == "read_returned" + ) + assert ( + classify_next(returned, _call("Read", file_path="/w/src/cgis/pipeline.py"), "/w") + == "read_other" + ) + assert classify_next(returned, _call("Grep", pattern="x"), "/w") == "searched" + assert classify_next(returned, _call("TodoWrite"), "/w") == "other" + + +def test_sufficiency_counts_only_main_thread_cgis_calls() -> None: + t = parse_transcript(stub.cgis_session()) + assert sufficiency(t) == {"read_returned": 1, "searched": 1} + + +def test_allocation_is_the_share_of_named_files_the_answer_used() -> None: + t = parse_transcript(stub.cgis_session()) + # Named: cgis/cli.py, cgis/guardian/collector.py, cgis/query/render/mermaid.py. + assert allocation(t, ["src/cgis/cli.py"]) == 1 / 3 + assert allocation(Transcript(), ["a.py"]) is None + + +def test_run_metrics_on_a_clean_treatment_session() -> None: + t = parse_transcript(stub.cgis_session()) + m = run_metrics(t, ["src/cgis/cli.py", "src/cgis/extractors/registry.py"]) + assert m.total_tool_calls == 5 + assert m.cgis_calls == 2 + assert m.tool_calls["Read"] == 2 + assert m.files_read == 2 # sub-agent reads count too: the session paid for them + assert m.bytes_read == len("def structure(): ...\n") + assert m.cli_attempts == 0 + assert m.contaminated is False + + +def test_a_blocked_attempt_is_counted_and_one_that_got_output_contaminates() -> None: + blocked = parse_transcript( + stub.lines( + stub.tool_use("b", "Bash", {"command": "cgis overview"}), + stub.tool_result("b", "x", is_error=True), + ) + ) + leaked = parse_transcript( + stub.lines( + stub.tool_use("b", "Bash", {"command": "cgis overview"}), + stub.tool_result("b", "graph!"), + ) + ) + assert run_metrics(blocked, []).cli_attempts == 1 + assert run_metrics(blocked, []).contaminated is False + assert run_metrics(leaked, []).contaminated is True From 9843df7f4f234234a7d71dfc5a44c80ebf373aa0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 13:23:54 +0000 Subject: [PATCH 2/5] fix(bench): survive session timeouts, harden answer and arg parsing Addresses review on #547: a timed-out session keeps its partial transcript instead of ending the batch; entry points resolve with .exe on Windows; --repo strips whitespace; a missing tasks directory fails loudly; the answer is the last JSON block that carries symbols or files. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L7RD9DDX3xidUCfgBEtRDa --- scripts/agent_ab.py | 52 +++++++++++++++++++++--------- src/cgis/bench/agent_task.py | 27 ++++++++++++---- src/cgis/bench/transcript.py | 6 +++- tests/unit/test_agent_ab_script.py | 26 +++++++++++++++ tests/unit/test_agent_ab_task.py | 13 ++++++++ 5 files changed, 101 insertions(+), 23 deletions(-) diff --git a/scripts/agent_ab.py b/scripts/agent_ab.py index 311e469d..05e2f522 100644 --- a/scripts/agent_ab.py +++ b/scripts/agent_ab.py @@ -50,6 +50,12 @@ _BENCH_DIR = _REPO_ROOT / "benchmarks" / "agent_ab" _BIN = Path(sys.executable).parent + +def bin_path(name: str) -> str: + """An entry point installed next to this interpreter (`.exe` on Windows).""" + return str(_BIN / (f"{name}.exe" if sys.platform == "win32" else name)) + + #: Executables that would let an agent reach the graph without MCP. A PATH entry #: holding any of them is dropped; sqlite3 lives in /usr/bin and is left to the hook. _HIDDEN_EXECUTABLES = ("cgis", "cgis-mcp", "codegraph-brain", "uv", "uvx") @@ -95,7 +101,7 @@ def mcp_config(arm: Arm) -> dict[str, object]: """The `--mcp-config` document: empty for control, this checkout's server for cgis.""" if arm == "control": return {"mcpServers": {}} - return {"mcpServers": {"cgis": {"command": str(_BIN / "cgis-mcp"), "args": []}}} + return {"mcpServers": {"cgis": {"command": bin_path("cgis-mcp"), "args": []}}} def hook_settings() -> dict[str, object]: @@ -171,7 +177,7 @@ def ingest(worktree: Path, src_root: str) -> float: start = time.monotonic() try: subprocess.run( - [str(_BIN / "cgis"), "ingest", src_root, "--output", "graph.db"], + [bin_path("cgis"), "ingest", src_root, "--output", "graph.db"], cwd=worktree, check=True, capture_output=True, @@ -229,7 +235,8 @@ def repo_paths(pairs: Sequence[str]) -> dict[str, Path]: """`NAME=PATH` arguments as a mapping; this checkout is always `cgis`.""" repos = {"cgis": _REPO_ROOT} for pair in pairs: - name, sep, path = pair.partition("=") + raw_name, sep, raw_path = pair.partition("=") + name, path = raw_name.strip(), raw_path.strip() if not sep or not name or not path: _msg = f"--repo expects NAME=PATH, got {pair!r}" raise ValueError(_msg) @@ -237,6 +244,29 @@ def repo_paths(pairs: Sequence[str]) -> dict[str, Path]: return repos +def run_session(cmd: list[str], cwd: Path, timeout: int) -> tuple[str, int]: + """Run one session; a timeout keeps the partial transcript and returns code -1. + + A raised TimeoutExpired would end the whole batch, losing every run after it. + The partial row still lands, with `subtype` empty, so the timeout is visible. + """ + try: + proc = subprocess.run( + cmd, + cwd=cwd, + env=agent_env(dict(os.environ)), + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + print(f"session timed out after {timeout}s: {cwd}", file=sys.stderr) + partial = exc.stdout or "" + return (partial.decode("utf-8", "replace") if isinstance(partial, bytes) else partial), -1 + return proc.stdout, proc.returncode + + def run_one(task: AgentTask, arm: Arm, run: int, repo: Path, args: argparse.Namespace) -> None: """One session: worktree → (ingest) → claude -p → transcript → results line.""" with worktree_at(task.sha, repo) as wt, tempfile.TemporaryDirectory(prefix="ab-") as tmp: @@ -256,31 +286,23 @@ def run_one(task: AgentTask, arm: Arm, run: int, repo: Path, args: argparse.Name print(" " + " ".join([*cmd[:2], "", *cmd[3:]])) return start = time.monotonic() - proc = subprocess.run( - cmd, - cwd=wt, - env=agent_env(dict(os.environ)), - capture_output=True, - text=True, - timeout=args.timeout, - check=False, - ) + stdout, returncode = run_session(cmd, wt, args.timeout) wall_s = time.monotonic() - start transcript_path = args.transcripts / task.id / f"{arm}-{run}.jsonl" transcript_path.parent.mkdir(parents=True, exist_ok=True) - transcript_path.write_text(proc.stdout, encoding="utf-8") + transcript_path.write_text(stdout, encoding="utf-8") meta: dict[str, object] = { "model": args.model, "effort": args.effort, "cgis_sha": _git("rev-parse", "HEAD"), - "returncode": proc.returncode, + "returncode": returncode, "ingest_s": round(ingest_s, 2), "wall_s": round(wall_s, 2), "transcript": str(transcript_path.relative_to(_REPO_ROOT)) if transcript_path.is_relative_to(_REPO_ROOT) else str(transcript_path), } - row = results_row(task, arm, run, proc.stdout.splitlines(), meta) + row = results_row(task, arm, run, stdout.splitlines(), meta) _append_jsonl(args.results, row) print( f"[{task.id} {arm} #{run}] recall={row['recall']:.2f} cost=${row['cost_usd']:.3f} " diff --git a/src/cgis/bench/agent_task.py b/src/cgis/bench/agent_task.py index 2ce71b34..1c04dde9 100644 --- a/src/cgis/bench/agent_task.py +++ b/src/cgis/bench/agent_task.py @@ -111,6 +111,9 @@ def load_task(path: Path) -> AgentTask: def load_tasks(directory: Path) -> list[AgentTask]: """Every `*.yaml` task in a directory, sorted by id; ids must be unique.""" + if not directory.is_dir(): + _msg = f"tasks directory not found: {directory}" + raise FileNotFoundError(_msg) tasks = sorted((load_task(p) for p in directory.glob("*.yaml")), key=lambda t: t.id) ids = [t.id for t in tasks] duplicates = sorted({i for i in ids if ids.count(i) > 1}) @@ -120,16 +123,13 @@ def load_tasks(directory: Path) -> list[AgentTask]: return tasks -def extract_answer(text: str) -> AgentAnswer | None: - """The last fenced JSON block in `text`, or None when there is none or it is malformed.""" - blocks = _JSON_BLOCK.findall(text) - if not blocks: - return None +def _answer_from_block(block: str) -> AgentAnswer | None: + """One fenced block as an AgentAnswer, or None when it is not one.""" try: - data = json.loads(blocks[-1]) + data = json.loads(block) except json.JSONDecodeError: return None - if not isinstance(data, dict): + if not isinstance(data, dict) or not data.keys() & {"symbols", "files"}: return None try: return AgentAnswer.model_validate(data) @@ -137,6 +137,19 @@ def extract_answer(text: str) -> AgentAnswer | None: return None +def extract_answer(text: str) -> AgentAnswer | None: + """The last fenced JSON block in `text` that is an answer, or None. + + Searched from the end, so an example block quoted earlier in the answer, or a + non-answer block after it, does not hide the real one. + """ + for block in reversed(_JSON_BLOCK.findall(text)): + answer = _answer_from_block(block) + if answer is not None: + return answer + return None + + def normalize_symbol(name: str) -> str: """Fold the spellings agents use for a symbol into one dotted form. diff --git a/src/cgis/bench/transcript.py b/src/cgis/bench/transcript.py index de8b3038..d5246652 100644 --- a/src/cgis/bench/transcript.py +++ b/src/cgis/bench/transcript.py @@ -254,7 +254,11 @@ def sufficiency(transcript: Transcript) -> Counter[str]: def allocation(transcript: Transcript, answer_files: list[str]) -> float | None: - """Share of files named by cgis answers that the final answer relied on, pooled.""" + """Share of files named by cgis answers that the final answer relied on, pooled. + + Counted per call, not per distinct file: a file returned by two calls was paid + for twice, and allocation measures where the returned budget went. + """ named = 0 used = 0 for call in transcript.calls: diff --git a/tests/unit/test_agent_ab_script.py b/tests/unit/test_agent_ab_script.py index 9cbd5f55..e55abae6 100644 --- a/tests/unit/test_agent_ab_script.py +++ b/tests/unit/test_agent_ab_script.py @@ -249,3 +249,29 @@ def test_report_without_results_fails(tmp_path: Path) -> None: def test_fmt_renders_missing_values_as_a_dash() -> None: assert ab.fmt_cell(None) == "—" assert ab.fmt_cell(3) == "3" + + +def test_repo_paths_strips_whitespace(tmp_path: Path) -> None: + assert ab.repo_paths([f" owner-api = {tmp_path} "])["owner-api"] == tmp_path.resolve() + + +@pytest.mark.parametrize(("platform", "suffix"), [("linux", ""), ("win32", ".exe")]) +def test_bin_path_adds_exe_on_windows( + monkeypatch: pytest.MonkeyPatch, platform: str, suffix: str +) -> None: + monkeypatch.setattr(ab.sys, "platform", platform) + assert ab.bin_path("cgis").endswith("cgis" + suffix) + + +@pytest.mark.parametrize("partial", [None, "partial\n", b"bytes\n"]) +def test_a_timed_out_session_keeps_its_partial_transcript( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, partial: str | bytes | None +) -> None: + def timeout(cmd: list[str], **_k: object) -> None: + raise subprocess.TimeoutExpired(cmd, 5, output=partial) + + monkeypatch.setattr(ab.subprocess, "run", timeout) + stdout, code = ab.run_session(["claude"], tmp_path, 5) + assert code == -1 + expected = partial.decode() if isinstance(partial, bytes) else (partial or "") + assert stdout == expected diff --git a/tests/unit/test_agent_ab_task.py b/tests/unit/test_agent_ab_task.py index b5a9537f..d2f390d8 100644 --- a/tests/unit/test_agent_ab_task.py +++ b/tests/unit/test_agent_ab_task.py @@ -204,3 +204,16 @@ def test_unparseable_answer_is_flagged_and_scores_zero_precision() -> None: def test_task_without_required_items_has_full_recall() -> None: score = score_answer(_task(), _answer([], [])) assert score.recall == 1.0 + + +def test_extract_answer_skips_a_trailing_block_that_is_not_an_answer() -> None: + text = _answer(["real"], ["a/b.py"]) + '\n```json\n{"example": [1]}\n```\n```json\n[1]\n```' + answer = extract_answer(text) + assert answer is not None + assert answer.symbols == ["real"] + + +def test_missing_tasks_directory_fails_loudly(tmp_path: Path) -> None: + missing = tmp_path / "nope" + with pytest.raises(FileNotFoundError, match="tasks directory not found"): + load_tasks(missing) From 64fa87c8ef0094c26db373214a26ac3874dc49f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 13:25:17 +0000 Subject: [PATCH 3/5] fix(bench): linear-time JSON fence and path scanning (Sonar S8786) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L7RD9DDX3xidUCfgBEtRDa --- src/cgis/bench/agent_task.py | 17 +++++++++++++++-- src/cgis/bench/transcript.py | 8 ++++++-- tests/unit/test_agent_ab_task.py | 8 ++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/cgis/bench/agent_task.py b/src/cgis/bench/agent_task.py index 1c04dde9..2c4423be 100644 --- a/src/cgis/bench/agent_task.py +++ b/src/cgis/bench/agent_task.py @@ -29,7 +29,6 @@ "Do not modify any files." ) -_JSON_BLOCK = re.compile(r"```json\s*\n(.*?)```", re.S) _CALL_PARENS = re.compile(r"\(.*\)$") _LINE_SUFFIX = re.compile(r":\d+(?:-\d+)?$") @@ -137,13 +136,27 @@ def _answer_from_block(block: str) -> AgentAnswer | None: return None +def _json_blocks(text: str) -> list[str]: + """Bodies of the ```json fences in `text`, in order. + + Split on the fence rather than matched with a lazy regex: a `.*?` across the + whole answer backtracks super-linearly on an unclosed fence (Sonar S8786). + """ + blocks = [] + for fenced in text.split("```")[1::2]: + info, _, body = fenced.partition("\n") + if info.strip() == "json": + blocks.append(body) + return blocks + + def extract_answer(text: str) -> AgentAnswer | None: """The last fenced JSON block in `text` that is an answer, or None. Searched from the end, so an example block quoted earlier in the answer, or a non-answer block after it, does not hide the real one. """ - for block in reversed(_JSON_BLOCK.findall(text)): + for block in reversed(_json_blocks(text)): answer = _answer_from_block(block) if answer is not None: return answer diff --git a/src/cgis/bench/transcript.py b/src/cgis/bench/transcript.py index d5246652..94dbd9e2 100644 --- a/src/cgis/bench/transcript.py +++ b/src/cgis/bench/transcript.py @@ -36,7 +36,10 @@ ] _SEARCH_TOOLS = frozenset({"Grep", "Glob", "Bash"}) -_SOURCE_PATH = re.compile(r"[\w.\-/]+\.(?:py|tsx?|jsx?)\b") +#: Path-like tokens, then filtered by suffix: one character class with no overlapping +#: suffix group, so matching stays linear (Sonar S8786). +_PATH_TOKEN = re.compile(r"[\w./-]+") +_SOURCE_SUFFIXES = (".py", ".ts", ".tsx", ".js", ".jsx") class ToolCall(BaseModel, frozen=True): @@ -224,7 +227,8 @@ def _relative(path: str, cwd: str) -> str: def returned_files(call: ToolCall) -> set[str]: """Source paths a tool result names.""" - return set(_SOURCE_PATH.findall(call.result)) + tokens = (t.rstrip(".") for t in _PATH_TOKEN.findall(call.result)) + return {t for t in tokens if t.endswith(_SOURCE_SUFFIXES)} def classify_next(returned: set[str], following: ToolCall | None, cwd: str) -> Sufficiency: diff --git a/tests/unit/test_agent_ab_task.py b/tests/unit/test_agent_ab_task.py index d2f390d8..9d0ed10c 100644 --- a/tests/unit/test_agent_ab_task.py +++ b/tests/unit/test_agent_ab_task.py @@ -217,3 +217,11 @@ def test_missing_tasks_directory_fails_loudly(tmp_path: Path) -> None: missing = tmp_path / "nope" with pytest.raises(FileNotFoundError, match="tasks directory not found"): load_tasks(missing) + + +def test_an_unclosed_fence_is_not_an_answer() -> None: + assert extract_answer('```json\n{"symbols": ["a"]}\n' + "x" * 50_000) is None + + +def test_a_fence_with_another_language_is_ignored() -> None: + assert extract_answer('```python\n{"symbols": ["a"]}\n```') is None From bacd290c812bd8ca08847a6793133e512c1fb370 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 14:19:06 +0000 Subject: [PATCH 4/5] fix(bench): js symbol spellings, graph-file word boundary, normalized read count Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L7RD9DDX3xidUCfgBEtRDa --- src/cgis/bench/agent_task.py | 3 ++- src/cgis/bench/guard.py | 2 +- src/cgis/bench/transcript.py | 9 +++++++-- tests/unit/test_agent_ab_guard.py | 6 ++++++ tests/unit/test_agent_ab_task.py | 5 +++++ tests/unit/test_agent_ab_transcript.py | 15 +++++++++++++++ 6 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/cgis/bench/agent_task.py b/src/cgis/bench/agent_task.py index 2c4423be..8a017c54 100644 --- a/src/cgis/bench/agent_task.py +++ b/src/cgis/bench/agent_task.py @@ -172,7 +172,8 @@ def normalize_symbol(name: str) -> str: s = name.strip().strip("`").strip() s = _CALL_PARENS.sub("", s) s = s.replace("::", ".").replace(":", ".").replace("/", ".").replace("\\", ".") - s = s.replace(".py.", ".").replace(".ts.", ".").replace(".tsx.", ".") + for ext in (".py.", ".tsx.", ".ts.", ".jsx.", ".js."): + s = s.replace(ext, ".") return s.strip(".") diff --git a/src/cgis/bench/guard.py b/src/cgis/bench/guard.py index 6301d3aa..94753998 100644 --- a/src/cgis/bench/guard.py +++ b/src/cgis/bench/guard.py @@ -22,7 +22,7 @@ r"(?:^|[;&|(`]|\$\()\s*(?:\S*/)?(?:cgis|cgis-mcp|codegraph-brain|uvx|uv|sqlite3)\b(?![./-])" ) _PYTHON_IMPORT = re.compile(r"-m\s+cgis\b|\b(?:import|from)\s+cgis\b") -_GRAPH_FILE = re.compile(r"graph\.(?:db|json)\b") +_GRAPH_FILE = re.compile(r"\bgraph\.(?:db|json)\b") _PATH_KEYS = ("file_path", "path", "pattern", "notebook_path") diff --git a/src/cgis/bench/transcript.py b/src/cgis/bench/transcript.py index 94dbd9e2..a98bf574 100644 --- a/src/cgis/bench/transcript.py +++ b/src/cgis/bench/transcript.py @@ -26,7 +26,7 @@ from pydantic import BaseModel, Field -from cgis.bench.agent_task import file_matches +from cgis.bench.agent_task import file_matches, normalize_file from cgis.bench.guard import blocked_reason CGIS_TOOL_PREFIX = "mcp__cgis__" @@ -283,7 +283,12 @@ def run_metrics(transcript: Transcript, answer_files: list[str]) -> RunMetrics: tool_calls=dict(sorted(counts.items())), total_tool_calls=len(transcript.calls), cgis_calls=sum(1 for c in transcript.calls if is_cgis_call(c)), - files_read=len({str(c.input.get("file_path", "")) for c in reads}), + files_read=len( + { + normalize_file(_relative(str(c.input.get("file_path", "")), transcript.cwd)) + for c in reads + } + ), bytes_read=sum(len(c.result.encode("utf-8")) for c in reads), cli_attempts=len(attempts), contaminated=any(not c.is_error for c in attempts), diff --git a/tests/unit/test_agent_ab_guard.py b/tests/unit/test_agent_ab_guard.py index 3056c280..aa6c5e0a 100644 --- a/tests/unit/test_agent_ab_guard.py +++ b/tests/unit/test_agent_ab_guard.py @@ -86,3 +86,9 @@ def test_main_allows_ordinary_calls(monkeypatch: pytest.MonkeyPatch) -> None: def test_main_lets_malformed_events_through(monkeypatch: pytest.MonkeyPatch, stdin: str) -> None: """Fail open: a malformed event is still caught afterwards as contamination.""" assert _run_main(monkeypatch, stdin) == 0 + + +@pytest.mark.parametrize("path", ["docs/dependency_graph.json", "paragraph.db"]) +def test_other_files_ending_in_graph_names_are_allowed(path: str) -> None: + assert blocked_reason("Read", {"file_path": path}) is None + assert blocked_reason("Bash", {"command": f"cat {path}"}) is None diff --git a/tests/unit/test_agent_ab_task.py b/tests/unit/test_agent_ab_task.py index 9d0ed10c..ba5e7434 100644 --- a/tests/unit/test_agent_ab_task.py +++ b/tests/unit/test_agent_ab_task.py @@ -225,3 +225,8 @@ def test_an_unclosed_fence_is_not_an_answer() -> None: def test_a_fence_with_another_language_is_ignored() -> None: assert extract_answer('```python\n{"symbols": ["a"]}\n```') is None + + +def test_normalize_symbol_strips_js_extensions() -> None: + assert normalize_symbol("web/app.jsx::render") == "web.app.render" + assert normalize_symbol("web/util.js::f") == "web.util.f" diff --git a/tests/unit/test_agent_ab_transcript.py b/tests/unit/test_agent_ab_transcript.py index 1e8a7775..4b9b615f 100644 --- a/tests/unit/test_agent_ab_transcript.py +++ b/tests/unit/test_agent_ab_transcript.py @@ -123,3 +123,18 @@ def test_a_blocked_attempt_is_counted_and_one_that_got_output_contaminates() -> assert run_metrics(blocked, []).cli_attempts == 1 assert run_metrics(blocked, []).contaminated is False assert run_metrics(leaked, []).contaminated is True + + +def test_files_read_counts_one_file_however_its_path_is_spelled() -> None: + t = parse_transcript( + stub.lines( + stub.init(cwd="/w"), + stub.tool_use("a", "Read", {"file_path": "/w/src/m.py"}), + stub.tool_result("a", "x"), + stub.tool_use("b", "Read", {"file_path": "./src/m.py"}), + stub.tool_result("b", "x"), + stub.tool_use("c", "Read", {"file_path": "src/m.py"}), + stub.tool_result("c", "x"), + ) + ) + assert run_metrics(t, []).files_read == 1 From d8641ed0fefbcb130010c61729151f0d50f3e120 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 14:23:35 +0000 Subject: [PATCH 5/5] fix(bench): quote the hook interpreter path, accept an uppercase JSON fence Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L7RD9DDX3xidUCfgBEtRDa --- scripts/agent_ab.py | 3 ++- src/cgis/bench/agent_task.py | 2 +- tests/unit/test_agent_ab_script.py | 8 ++++++++ tests/unit/test_agent_ab_task.py | 6 ++++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/agent_ab.py b/scripts/agent_ab.py index 05e2f522..0a622002 100644 --- a/scripts/agent_ab.py +++ b/scripts/agent_ab.py @@ -24,6 +24,7 @@ import argparse import json import os +import shlex import shutil import statistics import subprocess @@ -106,7 +107,7 @@ def mcp_config(arm: Arm) -> dict[str, object]: def hook_settings() -> dict[str, object]: """The `--settings` document installing the guard hook on every tool call.""" - command = f"{sys.executable} -m cgis.bench.guard" + command = f"{shlex.quote(sys.executable)} -m cgis.bench.guard" return { "hooks": { "PreToolUse": [{"matcher": ".*", "hooks": [{"type": "command", "command": command}]}] diff --git a/src/cgis/bench/agent_task.py b/src/cgis/bench/agent_task.py index 8a017c54..7e39732a 100644 --- a/src/cgis/bench/agent_task.py +++ b/src/cgis/bench/agent_task.py @@ -145,7 +145,7 @@ def _json_blocks(text: str) -> list[str]: blocks = [] for fenced in text.split("```")[1::2]: info, _, body = fenced.partition("\n") - if info.strip() == "json": + if info.strip().lower() == "json": blocks.append(body) return blocks diff --git a/tests/unit/test_agent_ab_script.py b/tests/unit/test_agent_ab_script.py index e55abae6..f47de3d9 100644 --- a/tests/unit/test_agent_ab_script.py +++ b/tests/unit/test_agent_ab_script.py @@ -275,3 +275,11 @@ def timeout(cmd: list[str], **_k: object) -> None: assert code == -1 expected = partial.decode() if isinstance(partial, bytes) else (partial or "") assert stdout == expected + + +def test_the_hook_quotes_an_interpreter_path_with_spaces( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(ab.sys, "executable", "/opt/my env/bin/python") + hook = ab.hook_settings()["hooks"]["PreToolUse"][0]["hooks"][0] # type: ignore[index] + assert hook["command"] == "'/opt/my env/bin/python' -m cgis.bench.guard" diff --git a/tests/unit/test_agent_ab_task.py b/tests/unit/test_agent_ab_task.py index ba5e7434..42d4ae20 100644 --- a/tests/unit/test_agent_ab_task.py +++ b/tests/unit/test_agent_ab_task.py @@ -230,3 +230,9 @@ def test_a_fence_with_another_language_is_ignored() -> None: def test_normalize_symbol_strips_js_extensions() -> None: assert normalize_symbol("web/app.jsx::render") == "web.app.render" assert normalize_symbol("web/util.js::f") == "web.util.f" + + +def test_the_fence_language_is_case_insensitive() -> None: + answer = extract_answer('```JSON\n{"symbols": ["a"]}\n```') + assert answer is not None + assert answer.symbols == ["a"]