From 8aee532d944901725cda6a4cbfbc20f81109a07a Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 7 Aug 2026 14:54:48 +0100 Subject: [PATCH 1/4] fix(core): normalize claude-code MCP tool names to bare endpoint Claude Code names MCP tools `mcp____`, so the parser recorded `originalName` (the scorer-facing endpoint) as e.g. `mcp__supabase-mcp__query_logs`, while the Codex parser records the bare `query_logs`. A scorer doing `tc.endpoint === "query_logs"` therefore matches Codex but silently never matches Claude Code, so tool-selection evals false-fail on Claude Code for behavior that actually happened. Strip the `mcp____` prefix in the claude-code parser so the endpoint is agent-agnostic, consistent with Codex. The raw name is still passed to `normalizeToolName`, so `name` stays `tool_use`. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/agents/claude-code/parser.test.ts | 8 ++++--- .../core/src/agents/claude-code/parser.ts | 21 ++++++++++++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/packages/core/src/agents/claude-code/parser.test.ts b/packages/core/src/agents/claude-code/parser.test.ts index 515cb4d6..da63dce5 100644 --- a/packages/core/src/agents/claude-code/parser.test.ts +++ b/packages/core/src/agents/claude-code/parser.test.ts @@ -86,7 +86,9 @@ describe('claudeCodeParser', () => { expect(toolCalls.map((e) => e.tool?.name)).toEqual(['shell', 'tool_use']); expect(toolCalls.map((e) => e.tool?.originalName)).toEqual([ 'Bash', - 'mcp__supabase__search_docs', + // MCP tool: the `mcp____` prefix is stripped to the bare tool + // name so the endpoint matches across agents (e.g. Codex). + 'search_docs', ]); // the parser normalizes the shell command onto the event (args left raw) expect(toolCalls[0].tool?.command).toBe('ls -la'); @@ -192,7 +194,7 @@ describe('adaptTranscript', () => { ts: Date.parse('2026-06-18T10:00:00.000Z'), }, { - endpoint: 'mcp__supabase__search_docs', + endpoint: 'search_docs', body: { query: 'rls' }, name: 'tool_use', result: undefined, @@ -214,7 +216,7 @@ describe('adaptTranscript', () => { }, { type: 'tool_call', - name: 'mcp__supabase__search_docs', + name: 'search_docs', input: { query: 'rls' }, output: undefined, error: 'boom', diff --git a/packages/core/src/agents/claude-code/parser.ts b/packages/core/src/agents/claude-code/parser.ts index d0aef98a..f835d49d 100644 --- a/packages/core/src/agents/claude-code/parser.ts +++ b/packages/core/src/agents/claude-code/parser.ts @@ -53,6 +53,23 @@ const CLAUDE_CODE_TOOLS: AgentToolMap = { }, }; +/** + * Claude Code names MCP tools `mcp____`. Strip the + * `mcp____` prefix so the scorer-facing `originalName` is the bare tool + * name (e.g. `query_logs`), matching how the other agent parsers emit MCP tool + * names (Codex records the bare `tool` from its `mcp_tool_call` shape). Without + * this, the same logical MCP call has a different endpoint per agent, so a + * scorer's `tc.endpoint === 'query_logs'` silently never matches Claude Code. + * Non-MCP tools are returned unchanged. + */ +function bareToolName(name: string): string { + if (!name.startsWith('mcp__')) return name; + // ['mcp', '', ''] — rejoin trailing segments so a tool name + // that itself contains '__' survives. + const parts = name.split('__'); + return parts.length >= 3 ? parts.slice(2).join('__') : name; +} + /** * Claude Code's tool args → normalized fields. Owned here, not in shared: Read/ * Write/Edit carry the path in `file_path`, NotebookEdit in `notebook_path`, @@ -230,8 +247,10 @@ function recordToEvents(data: Record): TranscriptEvent[] { timestamp, type: 'tool_call', tool: { + // Pass the raw name to normalizeToolName so its `mcp__` → `tool_use` + // fallback still fires; expose the bare name as the endpoint. name: normalizeToolName(use.name, CLAUDE_CODE_TOOLS), - originalName: use.name, + originalName: bareToolName(use.name), id: use.id, args: use.input, }, From cc59f5a7915895055194354acdd0f7d42e66055a Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 7 Aug 2026 16:12:54 +0100 Subject: [PATCH 2/4] feat(core): structured tool-call identity (mcp/other + server) across parsers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the flat scorer-facing `endpoint` string with a structured `tool: ToolCall` — `{ kind: 'mcp'; server; toolName } | { kind: 'other'; toolName }` — produced by each agent parser from what its transcript encodes: - claude-code: split `mcp____` structurally. - codex: `mcp_tool_call` items carry the server (explicit `item.server`, else the sole configured MCP server from ParseContext); bare `item.tool` name. - opencode: `_` is lossy, so recover the server by matching the configured names' sanitized prefix (opencode's own `sanitize`) via a new optional `ParseContext.mcpServerNames`, threaded from the engine. - ai-sdk executor: attribute via a tool→server map built from the MCP handles. `toolName` is the bare, agent-agnostic name (no server prefix); `originalName` stays raw for tracing. This lets scorers disambiguate our MCP server's tool from a same-named native/hosted tool or another server's tool, e.g. `tc.tool.kind === 'mcp' && tc.tool.server === 'supabase-mcp' && tc.tool.toolName === 'query_logs'`. Supersedes the earlier claude-code-only prefix strip. Updates consumers (docs-results, resolve-database-001 scorer) and all parser tests; adds MCP attribution coverage for codex (explicit + sole-server) and opencode (prefix). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../EVAL.ts | 2 +- .../src/agents/claude-code/parser.test.ts | 15 +++-- .../core/src/agents/claude-code/parser.ts | 38 ++++++----- packages/core/src/agents/codex/parser.test.ts | 44 ++++++++++++- packages/core/src/agents/codex/parser.ts | 48 ++++++++++---- packages/core/src/agents/engine.ts | 6 +- .../core/src/agents/opencode/parser.test.ts | 30 +++++++-- packages/core/src/agents/opencode/parser.ts | 65 ++++++++++++++++--- packages/core/src/docs-results.test.ts | 19 +++++- packages/core/src/docs-results.ts | 6 +- packages/core/src/index.ts | 28 ++++++-- packages/core/src/parsers/adapt.ts | 11 +++- packages/core/src/parsers/types.ts | 14 +++- packages/core/src/skill-results.test.ts | 4 +- packages/core/src/transcript/types.ts | 22 ++++++- 15 files changed, 284 insertions(+), 68 deletions(-) diff --git a/evals/resolve-database-001-migration-history-mismatch/EVAL.ts b/evals/resolve-database-001-migration-history-mismatch/EVAL.ts index 25ac639e..f1d1b66c 100644 --- a/evals/resolve-database-001-migration-history-mismatch/EVAL.ts +++ b/evals/resolve-database-001-migration-history-mismatch/EVAL.ts @@ -352,7 +352,7 @@ function formatActionsForJudge(toolCalls: ToolCallRecord[]): string { outcome = `\n output: ${truncMiddle(res, 600)}`; } - return `#${i + 1} [${tc.endpoint}] ${truncHead(String(action), 300)}${outcome}`; + return `#${i + 1} [${tc.tool.toolName}] ${truncHead(String(action), 300)}${outcome}`; }) .join('\n'); } diff --git a/packages/core/src/agents/claude-code/parser.test.ts b/packages/core/src/agents/claude-code/parser.test.ts index da63dce5..0cec8505 100644 --- a/packages/core/src/agents/claude-code/parser.test.ts +++ b/packages/core/src/agents/claude-code/parser.test.ts @@ -84,11 +84,16 @@ describe('claudeCodeParser', () => { const toolCalls = events.filter((e) => e.type === 'tool_call'); expect(toolCalls.map((e) => e.tool?.name)).toEqual(['shell', 'tool_use']); + // originalName stays raw as the agent emitted it. expect(toolCalls.map((e) => e.tool?.originalName)).toEqual([ 'Bash', - // MCP tool: the `mcp____` prefix is stripped to the bare tool - // name so the endpoint matches across agents (e.g. Codex). - 'search_docs', + 'mcp__supabase__search_docs', + ]); + // `call` carries the agent-agnostic identity: MCP tools are split into + // server + bare toolName; built-ins are `other`. + expect(toolCalls.map((e) => e.tool?.call)).toEqual([ + { kind: 'other', toolName: 'Bash' }, + { kind: 'mcp', server: 'supabase', toolName: 'search_docs' }, ]); // the parser normalizes the shell command onto the event (args left raw) expect(toolCalls[0].tool?.command).toBe('ls -la'); @@ -185,7 +190,7 @@ describe('adaptTranscript', () => { it('builds tool calls keyed by the original tool name, with results paired in', () => { expect(adapted.toolCalls).toEqual([ { - endpoint: 'Bash', + tool: { kind: 'other', toolName: 'Bash' }, body: { command: 'ls -la' }, name: 'shell', command: 'ls -la', @@ -194,7 +199,7 @@ describe('adaptTranscript', () => { ts: Date.parse('2026-06-18T10:00:00.000Z'), }, { - endpoint: 'search_docs', + tool: { kind: 'mcp', server: 'supabase', toolName: 'search_docs' }, body: { query: 'rls' }, name: 'tool_use', result: undefined, diff --git a/packages/core/src/agents/claude-code/parser.ts b/packages/core/src/agents/claude-code/parser.ts index f835d49d..83b5e3ae 100644 --- a/packages/core/src/agents/claude-code/parser.ts +++ b/packages/core/src/agents/claude-code/parser.ts @@ -13,6 +13,7 @@ import type { ParsedTranscript, + ToolCall, TranscriptEvent, } from '../../transcript/types.js'; import type { AgentTranscriptParser } from '../../parsers/types.js'; @@ -54,20 +55,26 @@ const CLAUDE_CODE_TOOLS: AgentToolMap = { }; /** - * Claude Code names MCP tools `mcp____`. Strip the - * `mcp____` prefix so the scorer-facing `originalName` is the bare tool - * name (e.g. `query_logs`), matching how the other agent parsers emit MCP tool - * names (Codex records the bare `tool` from its `mcp_tool_call` shape). Without - * this, the same logical MCP call has a different endpoint per agent, so a - * scorer's `tc.endpoint === 'query_logs'` silently never matches Claude Code. - * Non-MCP tools are returned unchanged. + * Claude Code names MCP tools `mcp____` (fixture: + * `mcp__supabase-mcp__query_logs`). The `mcp__` marker and `__` delimiters make + * the server and bare tool name structurally recoverable, so scorers get an + * agent-agnostic `toolName` plus the originating `server`. Anything else is a + * built-in / native tool. */ -function bareToolName(name: string): string { - if (!name.startsWith('mcp__')) return name; - // ['mcp', '', ''] — rejoin trailing segments so a tool name - // that itself contains '__' survives. - const parts = name.split('__'); - return parts.length >= 3 ? parts.slice(2).join('__') : name; +function toolCall(rawName: string): ToolCall { + if (rawName.startsWith('mcp__')) { + // ['mcp', '', ''] — rejoin trailing segments so a tool + // name that itself contains '__' survives. + const parts = rawName.split('__'); + if (parts.length >= 3) { + return { + kind: 'mcp', + server: parts[1]!, + toolName: parts.slice(2).join('__'), + }; + } + } + return { kind: 'other', toolName: rawName }; } /** @@ -247,10 +254,9 @@ function recordToEvents(data: Record): TranscriptEvent[] { timestamp, type: 'tool_call', tool: { - // Pass the raw name to normalizeToolName so its `mcp__` → `tool_use` - // fallback still fires; expose the bare name as the endpoint. name: normalizeToolName(use.name, CLAUDE_CODE_TOOLS), - originalName: bareToolName(use.name), + originalName: use.name, + call: toolCall(use.name), id: use.id, args: use.input, }, diff --git a/packages/core/src/agents/codex/parser.test.ts b/packages/core/src/agents/codex/parser.test.ts index ddb52ebd..1db0cef3 100644 --- a/packages/core/src/agents/codex/parser.test.ts +++ b/packages/core/src/agents/codex/parser.test.ts @@ -77,7 +77,7 @@ describe('codexParser', () => { expect(adapted.steps).toBe(2); // two agent_message turns expect(adapted.toolCalls).toEqual([ { - endpoint: 'command_execution', + tool: { kind: 'other', toolName: 'command_execution' }, body: { command: "/bin/zsh -lc 'echo hi'" }, name: 'shell', command: "/bin/zsh -lc 'echo hi'", @@ -86,7 +86,7 @@ describe('codexParser', () => { ts: 0, }, { - endpoint: 'file_change', + tool: { kind: 'other', toolName: 'file_change' }, body: { changes: [{ path: '/work/note.txt', kind: 'add' }] }, name: 'file_write', path: '/work/note.txt', @@ -168,6 +168,46 @@ describe('codexParser', () => { expect(result?.tool?.originalName).toBe('search_docs'); }); + it('attributes an mcp_tool_call to its server (explicit field or sole configured server)', () => { + const withServer = JSON.stringify({ + type: 'item.completed', + item: { + id: 'm1', + type: 'mcp_tool_call', + server: 'supabase-mcp', + tool: 'query_logs', + result: 'ok', + }, + }); + const explicit = codexParser + .parseTranscript(withServer) + .events.find((e) => e.type === 'tool_call'); + expect(explicit?.tool?.call).toEqual({ + kind: 'mcp', + server: 'supabase-mcp', + toolName: 'query_logs', + }); + + // No server field: fall back to the sole configured MCP server. + const noServer = JSON.stringify({ + type: 'item.completed', + item: { + id: 'm2', + type: 'mcp_tool_call', + tool: 'query_logs', + result: 'ok', + }, + }); + const fallback = codexParser + .parseTranscript(noServer, { mcpServerNames: ['supabase-mcp'] }) + .events.find((e) => e.type === 'tool_call'); + expect(fallback?.tool?.call).toEqual({ + kind: 'mcp', + server: 'supabase-mcp', + toolName: 'query_logs', + }); + }); + it("keeps a web_search item's action, which says what the hosted tool did", () => { const url = 'https://supabase.com/changelog.md'; const stream = JSON.stringify({ diff --git a/packages/core/src/agents/codex/parser.ts b/packages/core/src/agents/codex/parser.ts index 452bb8c8..9fd18375 100644 --- a/packages/core/src/agents/codex/parser.ts +++ b/packages/core/src/agents/codex/parser.ts @@ -21,9 +21,13 @@ import { isRecord, parseJsonlRecords } from '../../json.js'; import type { ParsedTranscript, + ToolCall, TranscriptEvent, } from '../../transcript/types.js'; -import type { AgentTranscriptParser } from '../../parsers/types.js'; +import type { + AgentTranscriptParser, + ParseContext, +} from '../../parsers/types.js'; import { normalizeToolName, type AgentToolMap, @@ -96,12 +100,16 @@ function toolCallPair( args: Record, result: unknown, success: boolean | undefined, - normalized: ExtractedArgs = {} + normalized: ExtractedArgs = {}, + // MCP items pass an explicit call identity; native items default to `other` + // with the item-type name as the bare tool name. + call: ToolCall = { kind: 'other', toolName: originalName } ): TranscriptEvent[] { const name = normalizeToolName(originalName, CODEX_TOOLS); const tool: NonNullable = { name, originalName, + call, id, args, }; @@ -125,7 +133,10 @@ function loadedSkillsFromCodexCall( return []; } -function itemToEvents(item: Record): TranscriptEvent[] { +function itemToEvents( + item: Record, + soleServer?: string +): TranscriptEvent[] { const id = str(item.id) ?? ''; const itemType = str(item.type); @@ -169,15 +180,23 @@ function itemToEvents(item: Record): TranscriptEvent[] { } case 'mcp_tool_call': { // Shape not pinned across versions — be defensive about field names and - // treat a missing status as unknown (not success). - const tool = + // treat a missing status as unknown (not success). `item.tool` is the + // bare tool name; `item.server` names the MCP server when present. + const bare = str(item.tool) ?? str(item.name) ?? str(item.server) ?? 'mcp_tool_call'; + // Prefer the explicit server field; fall back to the sole configured MCP + // server when the (unpinned) shape omits it. + const server = str(item.server) ?? soleServer; return toolCallPair( id, - tool, + bare, item, item.result ?? item.output, - statusSuccess(item.status) + statusSuccess(item.status), + {}, + server + ? { kind: 'mcp', server, toolName: bare } + : { kind: 'other', toolName: bare } ); } case 'web_search': { @@ -198,10 +217,13 @@ function itemToEvents(item: Record): TranscriptEvent[] { } } -function recordToEvents(data: Record): TranscriptEvent[] { +function recordToEvents( + data: Record, + soleServer?: string +): TranscriptEvent[] { switch (data.type) { case 'item.completed': - return isRecord(data.item) ? itemToEvents(data.item) : []; + return isRecord(data.item) ? itemToEvents(data.item, soleServer) : []; case 'turn.failed': case 'error': { const message = @@ -215,12 +237,16 @@ function recordToEvents(data: Record): TranscriptEvent[] { } export const codexParser: AgentTranscriptParser = { - parseTranscript(raw: string): ParsedTranscript { + parseTranscript(raw: string, ctx?: ParseContext): ParsedTranscript { const { records, errors } = parseJsonlRecords(raw); + // When the mcp_tool_call shape omits the server, attribute to the sole + // configured MCP server if there's exactly one. + const soleServer = + ctx?.mcpServerNames?.length === 1 ? ctx.mcpServerNames[0] : undefined; const events: TranscriptEvent[] = []; for (const record of records) { try { - events.push(...recordToEvents(record)); + events.push(...recordToEvents(record, soleServer)); } catch (e) { errors.push(e instanceof Error ? e.message : String(e)); } diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index de61c1fa..41035a46 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -106,7 +106,11 @@ export function createCliAgent( timeoutSec: args.timeoutSec, }); - const { events } = raw ? parser.parseTranscript(raw) : { events: [] }; + const { events } = raw + ? parser.parseTranscript(raw, { + mcpServerNames: Object.keys(args.mcpServers ?? {}), + }) + : { events: [] }; const adapted = adaptTranscript(events); // Surface run failures that would otherwise be invisible in results diff --git a/packages/core/src/agents/opencode/parser.test.ts b/packages/core/src/agents/opencode/parser.test.ts index 654607e0..cb7faa64 100644 --- a/packages/core/src/agents/opencode/parser.test.ts +++ b/packages/core/src/agents/opencode/parser.test.ts @@ -70,10 +70,28 @@ describe('opencodeParser', () => { state: { status: 'completed', input: { schemas: ['public'] } }, }, }); - const { events } = opencodeParser.parseTranscript(record); - const call = events.find((e) => e.type === 'tool_call'); - expect(call?.tool?.name).toBe('tool_use'); - expect(call?.tool?.originalName).toBe('supabase-mcp_list_tables'); + // Without the configured server name, the `_` join is + // unsplittable, so the server is unattributed (`other`). + const bare = opencodeParser.parseTranscript(record); + const bareCall = bare.events.find((e) => e.type === 'tool_call'); + expect(bareCall?.tool?.name).toBe('tool_use'); + expect(bareCall?.tool?.originalName).toBe('supabase-mcp_list_tables'); + expect(bareCall?.tool?.call).toEqual({ + kind: 'other', + toolName: 'supabase-mcp_list_tables', + }); + + // Given the configured server name, the prefix is stripped and the call is + // attributed to that MCP server with the bare tool name. + const attributed = opencodeParser.parseTranscript(record, { + mcpServerNames: ['supabase-mcp'], + }); + const call = attributed.events.find((e) => e.type === 'tool_call'); + expect(call?.tool?.call).toEqual({ + kind: 'mcp', + server: 'supabase-mcp', + toolName: 'list_tables', + }); }); it('maps bash + write to canonical tool calls, paired with results by callID', () => { @@ -106,7 +124,7 @@ describe('opencodeParser', () => { expect(adapted.steps).toBe(2); // two assistant text turns expect(adapted.toolCalls).toEqual([ { - endpoint: 'bash', + tool: { kind: 'other', toolName: 'bash' }, body: { command: 'ls -la', description: 'List files' }, name: 'shell', command: 'ls -la', @@ -115,7 +133,7 @@ describe('opencodeParser', () => { ts: 1782295624290, // epoch ms preserved through toISO -> parseTs }, { - endpoint: 'write', + tool: { kind: 'other', toolName: 'write' }, body: { filePath: '/work/note.txt', content: 'hi' }, name: 'file_write', path: '/work/note.txt', diff --git a/packages/core/src/agents/opencode/parser.ts b/packages/core/src/agents/opencode/parser.ts index 4c3fa21a..0a2e2909 100644 --- a/packages/core/src/agents/opencode/parser.ts +++ b/packages/core/src/agents/opencode/parser.ts @@ -23,9 +23,13 @@ import { isRecord, parseJsonlRecords } from '../../json.js'; import type { ParsedTranscript, + ToolCall, TranscriptEvent, } from '../../transcript/types.js'; -import type { AgentTranscriptParser } from '../../parsers/types.js'; +import type { + AgentTranscriptParser, + ParseContext, +} from '../../parsers/types.js'; import { normalizeToolName, type AgentToolMap, @@ -37,6 +41,38 @@ import { type ExtractedArgs, } from '../../parsers/shared/extract.js'; +/** + * opencode registers MCP tools as `sanitize(server) + "_" + sanitize(tool)` + * (opencode `packages/opencode/src/mcp/catalog.ts` @ v1.18.5), where + * `sanitize` replaces any char outside `[A-Za-z0-9_-]` with `_`. The join is a + * single `_` and both halves can contain `_`, so the split point is only + * recoverable by matching a known server name — which the harness supplies via + * `ParseContext.mcpServerNames`. Mirror opencode's `sanitize` so our prefix + * matches the one it built. + */ +const sanitizeServer = (value: string) => value.replace(/[^a-zA-Z0-9_-]/g, '_'); + +function mcpToolCall(originalName: string, mcpServerNames: string[]): ToolCall { + // Longest sanitized prefix first, so a server whose name is a prefix of + // another doesn't shadow the more specific match. + const byLongestPrefix = [...mcpServerNames].sort( + (a, b) => sanitizeServer(b).length - sanitizeServer(a).length + ); + for (const server of byLongestPrefix) { + const prefix = `${sanitizeServer(server)}_`; + if (originalName.startsWith(prefix)) { + return { + kind: 'mcp', + server, + toolName: originalName.slice(prefix.length), + }; + } + } + // Unmapped and unattributable: a custom tool, or an MCP server we weren't + // told about. Can't claim a server, so it's `other`. + return { kind: 'other', toolName: originalName }; +} + /** * opencode's tool names → canonical names. opencode uses lowercase built-in tool * names. Owned here, not in shared. MCP tools arrive under their server name and @@ -107,7 +143,8 @@ function partToEvents( type: string, part: Record, timestamp: string | undefined, - raw: unknown + raw: unknown, + mcpServerNames: string[] ): TranscriptEvent[] { switch (type) { case 'text': { @@ -136,15 +173,21 @@ function partToEvents( const status = str(state.status); const metadata = isRecord(state.metadata) ? state.metadata : undefined; // The builtin tool set is fully enumerated in OPENCODE_TOOLS, so any - // unmapped name is an MCP/custom tool (`_`, which the - // shared `mcp__` fallback doesn't recognize). + // unmapped name is an MCP/custom tool (`_`). Builtins are + // `other`; unmapped names are attributed to a configured MCP server by + // prefix (see mcpToolCall). const mapped = normalizeToolName(originalName, OPENCODE_TOOLS); - const name = mapped === 'unknown' ? 'tool_use' : mapped; + const isBuiltin = mapped !== 'unknown'; + const name = isBuiltin ? mapped : 'tool_use'; + const call: ToolCall = isBuiltin + ? { kind: 'other', toolName: originalName } + : mcpToolCall(originalName, mcpServerNames); const normalized: ExtractedArgs = extractArgs(args, OPENCODE_ARG_FIELDS); const tool: NonNullable = { name, originalName, + call, id, args, }; @@ -197,7 +240,10 @@ function loadedSkillsFromOpencodeCall( return []; } -function recordToEvents(data: Record): TranscriptEvent[] { +function recordToEvents( + data: Record, + mcpServerNames: string[] +): TranscriptEvent[] { const type = str(data.type); if (!type) return []; const timestamp = toISO(data.timestamp); @@ -221,16 +267,17 @@ function recordToEvents(data: Record): TranscriptEvent[] { const part = isRecord(data.part) ? data.part : undefined; if (!part) return []; - return partToEvents(type, part, timestamp, data); + return partToEvents(type, part, timestamp, data, mcpServerNames); } export const opencodeParser: AgentTranscriptParser = { - parseTranscript(raw: string): ParsedTranscript { + parseTranscript(raw: string, ctx?: ParseContext): ParsedTranscript { const { records, errors } = parseJsonlRecords(raw); + const mcpServerNames = ctx?.mcpServerNames ?? []; const events: TranscriptEvent[] = []; for (const record of records) { try { - events.push(...recordToEvents(record)); + events.push(...recordToEvents(record, mcpServerNames)); } catch (e) { errors.push(e instanceof Error ? e.message : String(e)); } diff --git a/packages/core/src/docs-results.test.ts b/packages/core/src/docs-results.test.ts index 2ac7d1dd..a1cd6aef 100644 --- a/packages/core/src/docs-results.test.ts +++ b/packages/core/src/docs-results.test.ts @@ -5,15 +5,30 @@ import { } from './docs-results.js'; import type { ToolCallRecord } from './index.js'; +/** Builds a `ToolCall` from a raw agent tool name (mirrors claude-code's `mcp__server__tool`). */ +function toToolCall(rawName: string): ToolCallRecord['tool'] { + if (rawName.startsWith('mcp__')) { + const parts = rawName.split('__'); + if (parts.length >= 3) { + return { + kind: 'mcp', + server: parts[1]!, + toolName: parts.slice(2).join('__'), + }; + } + } + return { kind: 'other', toolName: rawName }; +} + /** Builds the minimal tool call record needed by docs-result tests. */ function toolCall( - endpoint: string, + rawName: string, body: Record, options: Partial< Pick > = {} ): ToolCallRecord { - return { endpoint, body, ...options, ts: 0 }; + return { tool: toToolCall(rawName), body, ...options, ts: 0 }; } describe('buildDocsResult', () => { diff --git a/packages/core/src/docs-results.ts b/packages/core/src/docs-results.ts index b64c4df9..c8e9af39 100644 --- a/packages/core/src/docs-results.ts +++ b/packages/core/src/docs-results.ts @@ -173,7 +173,7 @@ function resultCharCount(result: unknown): number | undefined { /** True when a tool call is one of the channels docs activation tracks (worth rehydrating if truncated). */ function isDocsRelatedCall(call: ToolCallRecord): boolean { return ( - call.endpoint.endsWith('search_docs') || + call.tool.toolName === 'search_docs' || call.name === 'web_fetch' || call.name === 'web_search' || shellFetchUrls(shellCommand(call)).length > 0 @@ -228,9 +228,9 @@ export function buildDocsResult(toolCalls: ToolCallRecord[]): DocsResult { const calls: DocsCall[] = []; for (const call of toolCalls) { - const { endpoint, body, result } = call; + const { tool, body, result } = call; - if (endpoint.endsWith('search_docs')) { + if (tool.toolName === 'search_docs') { const graphqlQuery = extractGraphqlQuery(body); if (!graphqlQuery) continue; calls.push({ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 90a424d9..347b26ef 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -8,7 +8,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { basename, dirname, join } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; -import type { ToolName } from './transcript/types.js'; +import type { ToolCall, ToolName } from './transcript/types.js'; import { createClient, type SupabaseClient } from '@supabase/supabase-js'; import { createMCPClient } from '@ai-sdk/mcp'; import { Experimental_StdioMCPTransport as StdioMCPTransport } from '@ai-sdk/mcp/mcp-stdio'; @@ -181,7 +181,12 @@ export interface JudgeResult { } export interface ToolCallRecord { - endpoint: string; + /** + * Structured call identity: bare `toolName` plus its `mcp`/`other` source + * (and `server` for MCP). Replaces the old flat `endpoint` string so scorers + * can disambiguate an MCP server's tool from a same-named native/other tool. + */ + tool: ToolCall; body: Record; /** * Normalized, agent-agnostic views of common args, when the agent's parser @@ -642,6 +647,14 @@ export function aiSdkAgent(options: { const mcpHandles = args.mcpServers ? await createAiSdkTools(args.mcpServers) : []; + // Map each MCP tool name to its server so tool calls can be attributed. + // AI SDK keys tools by their bare MCP tool name (no server prefix). + const mcpToolServers = new Map(); + for (const handle of mcpHandles) { + for (const toolName of Object.keys(handle.tools)) { + mcpToolServers.set(toolName, handle.serverName); + } + } const toolCalls: ToolCallRecord[] = []; const transcript: TranscriptPart[] = [ { type: 'message', role: 'system', content: args.systemPrompt }, @@ -679,8 +692,12 @@ export function aiSdkAgent(options: { typeof input.command === 'string' ? input.command : undefined; + const toolName = event.toolCall.toolName; + const server = mcpToolServers.get(toolName); toolCalls.push({ - endpoint: event.toolCall.toolName, + tool: server + ? { kind: 'mcp', server, toolName } + : { kind: 'other', toolName }, body: input, command, loadedSkills, @@ -798,6 +815,7 @@ type ResolvedMcpServer = { }; type McpClientHandle = { + serverName: string; tools: ToolSet; close(): Promise; }; @@ -1239,7 +1257,7 @@ async function createAiSdkTools( const handles: McpClientHandle[] = []; try { - for (const server of Object.values(mcpServers)) { + for (const [serverName, server] of Object.entries(mcpServers)) { const transport = new StdioMCPTransport({ command: server.command, args: server.args, @@ -1248,7 +1266,7 @@ async function createAiSdkTools( }); const mcp = await createMCPClient({ transport }); const tools = await mcp.tools(); - handles.push({ tools, close: () => mcp.close() }); + handles.push({ serverName, tools, close: () => mcp.close() }); } } catch (err) { await closeMcpHandles(handles); diff --git a/packages/core/src/parsers/adapt.ts b/packages/core/src/parsers/adapt.ts index 622c0e23..58e916ca 100644 --- a/packages/core/src/parsers/adapt.ts +++ b/packages/core/src/parsers/adapt.ts @@ -9,7 +9,7 @@ */ import type { ToolCallRecord, TranscriptPart } from '../index.js'; -import type { TranscriptEvent } from '../transcript/types.js'; +import type { ToolCall, TranscriptEvent } from '../transcript/types.js'; export interface AdaptedTranscript { transcript: TranscriptPart[]; @@ -56,15 +56,20 @@ export function adaptTranscript(events: TranscriptEvent[]): AdaptedTranscript { const resolved = event.tool.id ? resultsById.get(event.tool.id) : undefined; + // Parsers set `call` on tool_call events; fall back defensively. + const call: ToolCall = event.tool.call ?? { + kind: 'other', + toolName: event.tool.originalName, + }; transcript.push({ type: 'tool_call', - name: event.tool.originalName, + name: call.toolName, input: body, output: resolved?.error === undefined ? resolved?.result : undefined, error: resolved?.error, }); toolCalls.push({ - endpoint: event.tool.originalName, + tool: call, body, // Normalized views the parser extracted, for agent-agnostic scorers. name: event.tool.name, diff --git a/packages/core/src/parsers/types.ts b/packages/core/src/parsers/types.ts index c08eabb9..434a30c3 100644 --- a/packages/core/src/parsers/types.ts +++ b/packages/core/src/parsers/types.ts @@ -11,6 +11,18 @@ import type { ParsedTranscript } from '../transcript/types.js'; +/** + * Optional run context for parsers. Some agents (OpenCode) name MCP tools + * `_` without a structural marker, so the server is only + * recoverable by matching the configured server names — which the harness + * knows but the transcript alone does not. Parsers whose format encodes the + * server (Claude Code, Codex) ignore this. + */ +export interface ParseContext { + /** Names of the MCP servers configured for this run. */ + mcpServerNames?: string[]; +} + export interface AgentTranscriptParser { - parseTranscript(raw: string): ParsedTranscript; + parseTranscript(raw: string, ctx?: ParseContext): ParsedTranscript; } diff --git a/packages/core/src/skill-results.test.ts b/packages/core/src/skill-results.test.ts index 6af25956..86af1d8a 100644 --- a/packages/core/src/skill-results.test.ts +++ b/packages/core/src/skill-results.test.ts @@ -4,12 +4,12 @@ import type { ToolCallRecord } from './index.js'; /** Builds the minimal tool call record needed by skill-result tests. */ function toolCall( - endpoint: string, + toolName: string, body: Record, options: Pick = {} ): ToolCallRecord { return { - endpoint, + tool: { kind: 'other', toolName }, body, ...options, ts: 0, diff --git a/packages/core/src/transcript/types.ts b/packages/core/src/transcript/types.ts index 01c73b64..abca11e9 100644 --- a/packages/core/src/transcript/types.ts +++ b/packages/core/src/transcript/types.ts @@ -28,6 +28,21 @@ export type ToolName = | 'tool_use' | 'unknown'; +/** + * A tool call's identity: its agent-agnostic `toolName` plus where it came + * from. `mcp` is attributed precisely (the parser knows the `server`), so + * scorers can tell our MCP server's `search_docs` apart from a same-named tool + * on another server or a native/hosted tool. Everything not attributable to a + * configured MCP server is `other` (agent built-ins, hosted tools like Codex's + * web_search, or custom tools). + * + * `toolName` is the bare tool name with any agent-specific MCP server prefix + * stripped (e.g. `query_logs`, not `mcp__supabase-mcp__query_logs`). + */ +export type ToolCall = + | { kind: 'mcp'; server: string; toolName: string } + | { kind: 'other'; toolName: string }; + /** A single normalized event in an agent transcript. */ export interface TranscriptEvent { /** ISO timestamp of the event, when the agent records one. */ @@ -42,8 +57,13 @@ export interface TranscriptEvent { tool?: { /** Canonical tool name. */ name: ToolName; - /** Original tool name as the agent emitted it (the scorer-facing endpoint). */ + /** Original tool name exactly as the agent emitted it (raw; kept for tracing). */ originalName: string; + /** + * Structured call identity (bare `toolName` + `mcp`/`other` source). Set on + * `tool_call` events; omitted on `tool_result` (correlated by `id`). + */ + call?: ToolCall; /** * Correlation id linking a `tool_call` to its later `tool_result`. Agents * that interleave the two (Claude Code's `tool_use_id`) set this so the From 94068497d86710f183b3dde762c0bdc1a970979d Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Mon, 10 Aug 2026 20:47:50 +0100 Subject: [PATCH 3/4] refactor(core): reuse claude-code's mcp__ tool-name split in docs-results tests docs-results.test.ts re-derived the mcp____ split instead of reusing claude-code/parser's parseClaudeCodeToolCall, which already owns that format. Export and reuse it instead of duplicating the parsing logic. --- .../core/src/agents/claude-code/parser.ts | 7 +++++-- packages/core/src/docs-results.test.ts | 21 ++++--------------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/packages/core/src/agents/claude-code/parser.ts b/packages/core/src/agents/claude-code/parser.ts index 83b5e3ae..dcc9249c 100644 --- a/packages/core/src/agents/claude-code/parser.ts +++ b/packages/core/src/agents/claude-code/parser.ts @@ -60,8 +60,11 @@ const CLAUDE_CODE_TOOLS: AgentToolMap = { * the server and bare tool name structurally recoverable, so scorers get an * agent-agnostic `toolName` plus the originating `server`. Anything else is a * built-in / native tool. + * + * Exported so tests elsewhere that need a Claude Code-shaped `ToolCall` (e.g. + * `docs-results.test.ts`) reuse this instead of re-deriving the split. */ -function toolCall(rawName: string): ToolCall { +export function parseClaudeCodeToolCall(rawName: string): ToolCall { if (rawName.startsWith('mcp__')) { // ['mcp', '', ''] — rejoin trailing segments so a tool // name that itself contains '__' survives. @@ -256,7 +259,7 @@ function recordToEvents(data: Record): TranscriptEvent[] { tool: { name: normalizeToolName(use.name, CLAUDE_CODE_TOOLS), originalName: use.name, - call: toolCall(use.name), + call: parseClaudeCodeToolCall(use.name), id: use.id, args: use.input, }, diff --git a/packages/core/src/docs-results.test.ts b/packages/core/src/docs-results.test.ts index a1cd6aef..681a8f79 100644 --- a/packages/core/src/docs-results.test.ts +++ b/packages/core/src/docs-results.test.ts @@ -1,26 +1,13 @@ import { describe, expect, it, vi } from 'vitest'; +import { parseClaudeCodeToolCall } from './agents/claude-code/parser.js'; import { buildDocsResult, rehydrateTruncatedDocsResults, } from './docs-results.js'; import type { ToolCallRecord } from './index.js'; -/** Builds a `ToolCall` from a raw agent tool name (mirrors claude-code's `mcp__server__tool`). */ -function toToolCall(rawName: string): ToolCallRecord['tool'] { - if (rawName.startsWith('mcp__')) { - const parts = rawName.split('__'); - if (parts.length >= 3) { - return { - kind: 'mcp', - server: parts[1]!, - toolName: parts.slice(2).join('__'), - }; - } - } - return { kind: 'other', toolName: rawName }; -} - -/** Builds the minimal tool call record needed by docs-result tests. */ +/** Builds the minimal tool call record needed by docs-result tests, from a raw + * agent tool name (Claude Code's `mcp__server__tool` shape). */ function toolCall( rawName: string, body: Record, @@ -28,7 +15,7 @@ function toolCall( Pick > = {} ): ToolCallRecord { - return { tool: toToolCall(rawName), body, ...options, ts: 0 }; + return { tool: parseClaudeCodeToolCall(rawName), body, ...options, ts: 0 }; } describe('buildDocsResult', () => { From 26982f15645af79736a30b16245eb0500ddfd468 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 14 Aug 2026 12:25:25 +0100 Subject: [PATCH 4/4] fix(core/codex): drop unreliable mcp_tool_call server/tool fallbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex's item.tool is non-optional per source, so falling back to item.server as a tool name was never actually reachable and misleading. Also drops the "sole configured MCP server" guess for a missing server field — better to surface kind: 'other' than silently attribute a call to a server that wasn't actually named. Addresses review comments from mattrossman on PR #178. --- packages/core/src/agents/codex/parser.test.ts | 9 +++--- packages/core/src/agents/codex/parser.ts | 32 +++++-------------- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/packages/core/src/agents/codex/parser.test.ts b/packages/core/src/agents/codex/parser.test.ts index 1db0cef3..532e5f4e 100644 --- a/packages/core/src/agents/codex/parser.test.ts +++ b/packages/core/src/agents/codex/parser.test.ts @@ -168,7 +168,7 @@ describe('codexParser', () => { expect(result?.tool?.originalName).toBe('search_docs'); }); - it('attributes an mcp_tool_call to its server (explicit field or sole configured server)', () => { + it('attributes an mcp_tool_call to its server', () => { const withServer = JSON.stringify({ type: 'item.completed', item: { @@ -188,7 +188,7 @@ describe('codexParser', () => { toolName: 'query_logs', }); - // No server field: fall back to the sole configured MCP server. + // No server field: falls back to `kind: 'other'` rather than guessing. const noServer = JSON.stringify({ type: 'item.completed', item: { @@ -199,11 +199,10 @@ describe('codexParser', () => { }, }); const fallback = codexParser - .parseTranscript(noServer, { mcpServerNames: ['supabase-mcp'] }) + .parseTranscript(noServer) .events.find((e) => e.type === 'tool_call'); expect(fallback?.tool?.call).toEqual({ - kind: 'mcp', - server: 'supabase-mcp', + kind: 'other', toolName: 'query_logs', }); }); diff --git a/packages/core/src/agents/codex/parser.ts b/packages/core/src/agents/codex/parser.ts index 9fd18375..f02afe10 100644 --- a/packages/core/src/agents/codex/parser.ts +++ b/packages/core/src/agents/codex/parser.ts @@ -24,10 +24,7 @@ import type { ToolCall, TranscriptEvent, } from '../../transcript/types.js'; -import type { - AgentTranscriptParser, - ParseContext, -} from '../../parsers/types.js'; +import type { AgentTranscriptParser } from '../../parsers/types.js'; import { normalizeToolName, type AgentToolMap, @@ -133,10 +130,7 @@ function loadedSkillsFromCodexCall( return []; } -function itemToEvents( - item: Record, - soleServer?: string -): TranscriptEvent[] { +function itemToEvents(item: Record): TranscriptEvent[] { const id = str(item.id) ?? ''; const itemType = str(item.type); @@ -182,11 +176,8 @@ function itemToEvents( // Shape not pinned across versions — be defensive about field names and // treat a missing status as unknown (not success). `item.tool` is the // bare tool name; `item.server` names the MCP server when present. - const bare = - str(item.tool) ?? str(item.name) ?? str(item.server) ?? 'mcp_tool_call'; - // Prefer the explicit server field; fall back to the sole configured MCP - // server when the (unpinned) shape omits it. - const server = str(item.server) ?? soleServer; + const bare = str(item.tool) ?? str(item.name) ?? 'mcp_tool_call'; + const server = str(item.server); return toolCallPair( id, bare, @@ -217,13 +208,10 @@ function itemToEvents( } } -function recordToEvents( - data: Record, - soleServer?: string -): TranscriptEvent[] { +function recordToEvents(data: Record): TranscriptEvent[] { switch (data.type) { case 'item.completed': - return isRecord(data.item) ? itemToEvents(data.item, soleServer) : []; + return isRecord(data.item) ? itemToEvents(data.item) : []; case 'turn.failed': case 'error': { const message = @@ -237,16 +225,12 @@ function recordToEvents( } export const codexParser: AgentTranscriptParser = { - parseTranscript(raw: string, ctx?: ParseContext): ParsedTranscript { + parseTranscript(raw: string): ParsedTranscript { const { records, errors } = parseJsonlRecords(raw); - // When the mcp_tool_call shape omits the server, attribute to the sole - // configured MCP server if there's exactly one. - const soleServer = - ctx?.mcpServerNames?.length === 1 ? ctx.mcpServerNames[0] : undefined; const events: TranscriptEvent[] = []; for (const record of records) { try { - events.push(...recordToEvents(record, soleServer)); + events.push(...recordToEvents(record)); } catch (e) { errors.push(e instanceof Error ? e.message : String(e)); }