From dbe46076bc52ae0d1925681fdd7c640f2b5b787e Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 8 Sep 2026 19:36:41 -0700 Subject: [PATCH] fix(codex): preserve unknown and failed tool outcomes --- docs/trace-analysts.md | 16 ++++ package.json | 2 +- src/adapters/codex-format.ts | 3 + src/adapters/codex.ts | 72 ++++++++++----- src/failure-followup.ts | 7 +- src/runtime-store.ts | 3 +- tests/adapters.test.ts | 20 ++--- tests/codex-tool-status.test.ts | 150 ++++++++++++++++++++++++++++++++ tests/failure-followup.test.ts | 18 +++- tests/runtime-store.test.ts | 11 +++ tests/upgrades.test.ts | 29 ++++-- 11 files changed, 283 insertions(+), 48 deletions(-) create mode 100644 tests/codex-tool-status.test.ts diff --git a/docs/trace-analysts.md b/docs/trace-analysts.md index 130b7e3..8c91a3c 100644 --- a/docs/trace-analysts.md +++ b/docs/trace-analysts.md @@ -56,6 +56,22 @@ Returned directories from another resumed Claude session are parsed and included The OpenInference file is the shared input for external engines. The original trace and exact cited span remain available for review. +## Codex tool outcomes + +Both function and custom tool outputs use the same status parser. +Explicit exit codes, error flags, and runner headers determine tool status. +Explicit failure takes precedence when structured fields conflict. +Text such as `error:`, `ENOENT`, or `command failed` inside stdout does not determine status. +An unmatched call, running process, or output without trustworthy status remains `UNSET`. +A recorded `wait_agent` timeout is a completed poll, not proof that the agent finished. +Tool output remains in `output.value`, with the existing size limit and truncation receipt. + +Error and retry counts use explicit failures. +Successful follow-up counts require explicit success; unknown follow-up outcomes remain `null` in the detailed report. +The runtime span projection leaves its optional status absent for `UNSET` spans. +Use the execution report for terminal run outcomes. +The older runtime store's required run status cannot represent unknown and is not completion evidence. + ## External engines External engines are optional tools that you install separately. diff --git a/package.json b/package.json index 9c1f38a..cbb2073 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/traces", - "version": "0.13.0", + "version": "0.13.1", "description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.", "type": "module", "license": "MIT", diff --git a/src/adapters/codex-format.ts b/src/adapters/codex-format.ts index 4e252a1..0b181b2 100644 --- a/src/adapters/codex-format.ts +++ b/src/adapters/codex-format.ts @@ -25,6 +25,9 @@ export interface CodexLine { input?: unknown call_id?: string output?: unknown + is_error?: boolean + isError?: boolean + error?: unknown event_id?: string turn_id?: string occurred_at_ms?: number diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index c3ae773..512ac62 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -102,8 +102,11 @@ function textOf(content: unknown): string { } function numericStatus(value: unknown): number | undefined { - if (typeof value === 'number' && Number.isFinite(value)) return value - if (typeof value === 'string' && /^-?\d+$/.test(value.trim())) return Number(value) + if (typeof value === 'number' && Number.isSafeInteger(value)) return value + if (typeof value === 'string' && /^-?\d+$/.test(value.trim())) { + const code = Number(value) + if (Number.isSafeInteger(code)) return code + } return undefined } @@ -120,22 +123,29 @@ function explicitOutputError(value: unknown, timeoutIsError = true): boolean | u if (value && typeof value === 'object') { const row = value as Record + let observedSuccess = false + for (const key of ['is_error', 'isError', 'error']) { + if (row[key] === true) return true + if (row[key] === false) observedSuccess = true + } for (const key of ['exit_code', 'exitCode']) { const code = numericStatus(row[key]) - if (code !== undefined && ('output' in row || 'chunk_id' in row || 'wall_time_seconds' in row)) { - return code !== 0 - } + if (code !== undefined && code !== 0) return true + if (code === 0) observedSuccess = true } for (const key of ['timed_out', 'timedOut']) { if (row[key] === true && timeoutIsError) return true } if (typeof row.succeeded === 'boolean' && ('value' in row || 'error' in row)) { - return !row.succeeded + if (!row.succeeded) return true + observedSuccess = true } if ((row.type === 'input_text' || row.type === 'text') && typeof row.text === 'string') { - return explicitOutputError(row.text, timeoutIsError) + const status = explicitOutputError(row.text, timeoutIsError) + if (status === true) return true + if (status === false) observedSuccess = true } - return undefined + return observedSuccess ? false : undefined } if (typeof value !== 'string') return undefined @@ -151,16 +161,16 @@ function explicitOutputError(value: unknown, timeoutIsError = true): boolean | u } const outputStart = text.indexOf('\nOutput:\n') const header = outputStart >= 0 ? text.slice(0, outputStart) : text - if (/^Chunk ID:/i.test(header)) { - const exitCode = header.match(/^Process exited with code\s+(-?\d+)\b/im)?.[1] - if (exitCode !== undefined) return Number(exitCode) !== 0 + if (/^(?:Chunk ID:|Process exited with code )/i.test(header)) { + const exitCode = numericStatus(header.match(/^Process exited with code[ \t]+(-?\d+)[ \t]*$/im)?.[1]) + if (exitCode !== undefined) return exitCode !== 0 } if (/^Script completed\s*\nWall time:/i.test(header)) return false if (/^Script failed\s*\nWall time:/i.test(header)) return true - const scriptExitCode = header.match(/^Script error:\s*\nExit code:\s*(-?\d+)\b/im)?.[1] - if (scriptExitCode !== undefined) return Number(scriptExitCode) !== 0 - const commandExitCode = text.match(/^Command failed with exit code\s+(-?\d+)\.?$/i)?.[1] - if (commandExitCode !== undefined) return Number(commandExitCode) !== 0 + const scriptExitCode = numericStatus(header.match(/^Script error:[ \t]*\r?\nExit code:[ \t]*(-?\d+)[ \t]*(?:\r?\n|$)/i)?.[1]) + if (scriptExitCode !== undefined) return scriptExitCode !== 0 + const commandExitCode = numericStatus(text.match(/^Command failed with exit code[ \t]+(-?\d+)\.?$/i)?.[1]) + if (commandExitCode !== undefined) return commandExitCode !== 0 if (/^[\s\S]*<\/tool_error>$/i.test(text)) return true return undefined } @@ -192,14 +202,29 @@ function isWaitAgentOperation(name: string): boolean { } /** Only protocol-level status fields count; arbitrary tool output may itself contain code or logs mentioning errors. */ -function outputStatus(name: string, output: unknown): { error: boolean; message: string; pollOutcome?: 'timeout' } { +function outputStatus( + name: string, + source: NonNullable, +): { status: OtlpSpan['status']; pollOutcome?: 'timeout' } { + const output = source.output const waitAgent = isWaitAgentOperation(name) - const error = explicitOutputError(output, !waitAgent) === true - const message = typeof output === 'string' ? output : JSON.stringify(output ?? '') + // Error metadata on the source envelope is authoritative. An `error` string + // inside arbitrary output remains domain data, not execution status. + const sourceError = source.is_error === true || source.isError === true + || (source.error !== undefined && source.error !== null && source.error !== false) + const outputError = explicitOutputError(output, !waitAgent) + const error = sourceError || outputError === true + const pollTimeout = !error && waitAgent && explicitTimeout(output) + const code = error ? 'ERROR' + : outputError === false || source.is_error === false || source.isError === false || pollTimeout ? 'OK' + : 'UNSET' + const message = sourceError ? source.error ?? output : output return { - error, - message: error ? message.slice(0, 500) : '', - ...(!error && waitAgent && explicitTimeout(output) ? { pollOutcome: 'timeout' as const } : {}), + status: { + code, + ...(error ? { message: (typeof message === 'string' ? message : JSON.stringify(message ?? '')).slice(0, 500) } : {}), + }, + ...(pollTimeout ? { pollOutcome: 'timeout' as const } : {}), } } @@ -782,6 +807,7 @@ export class CodexAdapter implements HarnessTraceAdapter { name: `tool.${name}`, kind: 'TOOL', startTime: ts, + status: 'UNSET', service: SERVICE, agent: SERVICE, tool: name, @@ -818,9 +844,9 @@ export class CodexAdapter implements HarnessTraceAdapter { const t = toolByCallId.get(l.payload.call_id ?? '') if (t) { const name = String(t.attributes['tool.name'] ?? '') - const { error, message, pollOutcome } = outputStatus(name, l.payload.output) + const { status, pollOutcome } = outputStatus(name, l.payload) closeSpanAt(t, ts) - t.status = error ? { code: 'ERROR', message } : { code: 'OK' } + t.status = status if (pollOutcome) t.attributes['traces.poll.outcome'] = pollOutcome recordToolOutput(t, l.payload.output) const operation = t.attributes['traces.codex.agent_operation'] diff --git a/src/failure-followup.ts b/src/failure-followup.ts index 0118888..2dd74d1 100644 --- a/src/failure-followup.ts +++ b/src/failure-followup.ts @@ -29,7 +29,7 @@ export interface FailureFollowUp { /** Null when the tool was never called again after the failure. */ followUpSpanId: string | null kind: FollowUpKind - /** Whether the follow-up call ended without an error; null when kind is 'none'. */ + /** Captured follow-up outcome; null when absent or unknown. */ followUpSucceeded: boolean | null } @@ -44,7 +44,7 @@ export interface FailureFollowUpReport { adapted: number /** Arguments were not captured or not comparable on one side of the pair. */ argsUnknown: number - /** Follow-ups that ended without an error, out of `followed`. */ + /** Follow-ups with explicit success, out of `followed`. */ followUpSucceeded: number /** Blind-retry count per tool, largest offender first when rendered. */ blindByTool: Record @@ -122,7 +122,8 @@ export function classifyFailureFollowUps(spans: readonly OtlpSpan[]): FailureFol failedSpanId: call.span.span_id, followUpSpanId: followUp?.span.span_id ?? null, kind, - followUpSucceeded: followUp ? followUp.span.status.code !== 'ERROR' : null, + followUpSucceeded: !followUp || followUp.span.status.code === 'UNSET' + ? null : followUp.span.status.code === 'OK', }) } } diff --git a/src/runtime-store.ts b/src/runtime-store.ts index cc60da4..29daa4d 100644 --- a/src/runtime-store.ts +++ b/src/runtime-store.ts @@ -93,7 +93,8 @@ export async function toRuntimeStore(spans: readonly OtlpSpan[]): Promise { expect(tools).toHaveLength(9) expect(tools.find((item) => item.span_id === codexSpanId('codex-tool-status', 'tool:read-source'))?.status).toEqual({ code: 'OK' }) expect(tools.find((item) => item.span_id === codexSpanId('codex-tool-status', 'tool:failed-command'))?.status.code).toBe('ERROR') - expect(tools.find((item) => item.span_id === codexSpanId('codex-tool-status', 'tool:domain-result'))?.status).toEqual({ code: 'OK' }) - expect(tools.find((item) => item.span_id === codexSpanId('codex-tool-status', 'tool:captured-log'))?.status).toEqual({ code: 'OK' }) - expect(tools.find((item) => item.span_id === codexSpanId('codex-tool-status', 'tool:captured-exit'))?.status).toEqual({ code: 'OK' }) + expect(tools.find((item) => item.span_id === codexSpanId('codex-tool-status', 'tool:domain-result'))?.status).toEqual({ code: 'UNSET' }) + expect(tools.find((item) => item.span_id === codexSpanId('codex-tool-status', 'tool:captured-log'))?.status).toEqual({ code: 'UNSET' }) + expect(tools.find((item) => item.span_id === codexSpanId('codex-tool-status', 'tool:captured-exit'))?.status).toEqual({ code: 'UNSET' }) expect(tools.find((item) => item.span_id === codexSpanId('codex-tool-status', 'tool:timed-out'))?.status.code).toBe('ERROR') expect(tools.find((item) => item.span_id === codexSpanId('codex-tool-status', 'tool:poll-timed-out'))).toMatchObject({ status: { code: 'OK' }, @@ -2206,7 +2206,7 @@ describe('codex current tool and subagent events', () => { { type: 'response_item', timestamp: '2026-07-11T09:00:04.200Z', - payload: { type: 'custom_tool_call_output', call_id: 'custom-2', output: 'Script completed' }, + payload: { type: 'custom_tool_call_output', call_id: 'custom-2', output: 'Script completed', is_error: false }, }, { type: 'response_item', @@ -2221,7 +2221,7 @@ describe('codex current tool and subagent events', () => { { type: 'response_item', timestamp: '2026-07-11T09:00:04.400Z', - payload: { type: 'custom_tool_call_output', call_id: 'custom-3', output: 'Script completed' }, + payload: { type: 'custom_tool_call_output', call_id: 'custom-3', output: 'Script completed', is_error: false }, }, { type: 'response_item', @@ -2236,7 +2236,7 @@ describe('codex current tool and subagent events', () => { { type: 'response_item', timestamp: '2026-07-11T09:00:04.600Z', - payload: { type: 'custom_tool_call_output', call_id: 'custom-4', output: 'Script completed' }, + payload: { type: 'custom_tool_call_output', call_id: 'custom-4', output: 'Script completed', is_error: false }, }, { type: 'response_item', @@ -2251,7 +2251,7 @@ describe('codex current tool and subagent events', () => { { type: 'response_item', timestamp: '2026-07-11T09:00:04.800Z', - payload: { type: 'function_call_output', call_id: 'blocking-1', output: 'Completed' }, + payload: { type: 'function_call_output', call_id: 'blocking-1', output: 'Completed', is_error: false }, }, { type: 'response_item', @@ -2266,7 +2266,7 @@ describe('codex current tool and subagent events', () => { { type: 'response_item', timestamp: '2026-07-11T09:00:05.000Z', - payload: { type: 'function_call_output', call_id: 'domain-wait-1', output: 'Completed' }, + payload: { type: 'function_call_output', call_id: 'domain-wait-1', output: 'Completed', is_error: false }, }, { type: 'response_item', @@ -2281,7 +2281,7 @@ describe('codex current tool and subagent events', () => { { type: 'response_item', timestamp: '2026-07-11T09:00:05.200Z', - payload: { type: 'custom_tool_call_output', call_id: 'malformed-input-1', output: 'Completed' }, + payload: { type: 'custom_tool_call_output', call_id: 'malformed-input-1', output: 'Completed', is_error: false }, }, { type: 'response_item', @@ -2296,7 +2296,7 @@ describe('codex current tool and subagent events', () => { { type: 'response_item', timestamp: '2026-07-11T09:00:05.400Z', - payload: { type: 'custom_tool_call_output', call_id: 'write-stdin-1', output: 'Completed' }, + payload: { type: 'custom_tool_call_output', call_id: 'write-stdin-1', output: 'Completed', is_error: false }, }, { type: 'event_msg', diff --git a/tests/codex-tool-status.test.ts b/tests/codex-tool-status.test.ts new file mode 100644 index 0000000..a059aa7 --- /dev/null +++ b/tests/codex-tool-status.test.ts @@ -0,0 +1,150 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, describe, expect, it } from 'vitest' +import { CodexAdapter } from '../src/adapters/codex.js' +import { summarizeSpanExecution } from '../src/execution.js' +import { runTraceInvestigation } from '../src/improvement.js' +import { runPipelines } from '../src/pipelines.js' +import { renderExecution } from '../src/report.js' +import { toRuntimeStore } from '../src/runtime-store.js' + +const dir = mkdtempSync(join(tmpdir(), 'traces-codex-status-')) +afterAll(() => rmSync(dir, { recursive: true, force: true })) + +async function parseOutputs( + variant: 'function' | 'custom', + outputs: Array<{ output?: unknown; source?: Record; pending?: boolean }>, +) { + const path = join(dir, `${variant}.jsonl`) + const events: Record[] = [ + { type: 'session_meta', payload: { id: 'status-session', cwd: '/fixture' } }, + { type: 'event_msg', payload: { type: 'task_started', turn_id: 'turn-1' } }, + ] + for (const [index, result] of outputs.entries()) { + events.push({ + type: 'response_item', + payload: variant === 'function' + ? { type: 'function_call', call_id: `call-${index}`, name: 'exec_command', arguments: '{"cmd":"pnpm test"}' } + : { type: 'custom_tool_call', call_id: `call-${index}`, name: 'exec', input: 'await tools.exec_command({ cmd: "pnpm test" })' }, + }) + if (!result.pending) { + events.push({ + type: 'response_item', + payload: { + type: variant === 'function' ? 'function_call_output' : 'custom_tool_call_output', + call_id: `call-${index}`, + output: result.output, + ...result.source, + }, + }) + } + } + events.push({ type: 'event_msg', payload: { type: 'task_complete', turn_id: 'turn-1' } }) + writeFileSync(path, events.map((event, index) => JSON.stringify({ + ...event, + timestamp: new Date(Date.UTC(2026, 8, 8, 0, 0, index)).toISOString(), + })).join('\n')) + return new CodexAdapter().parse({ harness: 'codex', sessionId: 'status-session', path, cwd: null, mtimeMs: 0 }) +} + +describe.each(['function', 'custom'] as const)('Codex %s tool outcomes', (variant) => { + it.each([ + { label: 'bare exit zero', output: { exit_code: 0 }, code: 'OK' }, + { label: 'bare nonzero exit', output: { exit_code: 1 }, code: 'ERROR' }, + { label: 'string integer exit', output: '{"exitCode":"-1"}', code: 'ERROR' }, + { label: 'explicit success with misleading stdout', output: { exit_code: 0, output: 'error: command failed ENOENT {"success":false}' }, code: 'OK' }, + { label: 'explicit failure with optimistic stdout', output: { exit_code: 1, output: 'success' }, code: 'ERROR' }, + { label: 'initial process header', output: 'Process exited with code 0\nOutput:\nerror: command failed ENOENT', code: 'OK' }, + { label: 'initial failed process header', output: 'Process exited with code 1\nOutput:\nsuccess', code: 'ERROR' }, + { label: 'initial script error receipt', output: 'Script error:\nExit code: 1\nOutput:\nsuccess', code: 'ERROR' }, + { label: 'initial script zero exit receipt', output: 'Script error:\nExit code: 0\nOutput:\nerror: source code', code: 'OK' }, + { label: 'script receipt with CRLF', output: 'Script error:\r\nExit code: -1\r\nOutput:\r\nsuccess', code: 'ERROR' }, + { label: 'standalone command failure receipt', output: 'Command failed with exit code 1.', code: 'ERROR' }, + { label: 'embedded script error', output: 'captured source\nScript error:\nExit code: 1', code: 'UNSET' }, + { label: 'fractional script exit', output: 'Script error:\nExit code: 0.5', code: 'UNSET' }, + { label: 'suffixed script exit', output: 'Script error:\nExit code: 1oops', code: 'UNSET' }, + { label: 'oversized script exit', output: 'Script error:\nExit code: 9007199254740992', code: 'UNSET' }, + { label: 'fractional process exit', output: 'Process exited with code 0.5', code: 'UNSET' }, + { label: 'oversized process exit', output: 'Process exited with code 9007199254740992', code: 'UNSET' }, + { label: 'fractional command exit', output: 'Command failed with exit code 0.5', code: 'UNSET' }, + { label: 'oversized command exit', output: 'Command failed with exit code 9007199254740992', code: 'UNSET' }, + { label: 'successful receipt before misleading script error', output: 'Process exited with code 0\nOutput:\nScript error:\nExit code: 1', code: 'OK' }, + { label: 'structured snake error flag', output: { is_error: true }, code: 'ERROR' }, + { label: 'structured camel error flag', output: { isError: true }, code: 'ERROR' }, + { label: 'boolean error flag', output: { error: true }, code: 'ERROR' }, + { label: 'explicit error overrides zero exit', output: { exit_code: 0, isError: true }, code: 'ERROR' }, + { label: 'explicit non-error flag', output: { is_error: false }, code: 'OK' }, + { label: 'nonzero exit overrides non-error flag', output: { exit_code: 1, is_error: false }, code: 'ERROR' }, + { label: 'wrapped text receipt', output: [{ type: 'input_text', text: '{"exit_code":0}' }], code: 'OK' }, + { label: 'failure among wrapped receipts', output: [{ type: 'input_text', text: '{"exit_code":0}' }, { type: 'input_text', text: '{"is_error":true}' }], code: 'ERROR' }, + { label: 'arbitrary stdout', output: 'error: command failed ENOENT {"success":false}', code: 'UNSET' }, + { label: 'embedded process header', output: 'captured file\nProcess exited with code 1\nnot a runner header', code: 'UNSET' }, + { label: 'domain error property', output: '{"error":"domain value","value":42}', code: 'UNSET' }, + { label: 'unrecognized object', output: { value: 42 }, code: 'UNSET' }, + { label: 'missing output', output: undefined, code: 'UNSET' }, + { label: 'running process receipt', output: { session_id: 123, exit_code: null, output: 'still running' }, code: 'UNSET' }, + { label: 'running script receipt', output: 'Script running with cell ID 123', code: 'UNSET' }, + { label: 'noninteger exit code', output: { exit_code: 0.5 }, code: 'UNSET' }, + ])('preserves $label', async ({ output, code }) => { + const spans = await parseOutputs(variant, [{ output }]) + const tool = spans.find((item) => item.attributes['openinference.span.kind'] === 'TOOL')! + expect(tool.status.code).toBe(code) + if (typeof output === 'string') expect(tool.attributes['output.value']).toBe(output) + const { store } = await toRuntimeStore(spans) + const runtimeTool = (await store.spans()).find((item) => item.kind === 'tool')! + expect(runtimeTool.status).toBe(code === 'UNSET' ? undefined : code === 'ERROR' ? 'error' : 'ok') + }) + + it.each([ + { is_error: true }, + { isError: true }, + { error: { message: 'source transport failure' } }, + { error: 'source transport failure' }, + ])('honors source failure metadata %j before successful output', async (source) => { + const spans = await parseOutputs(variant, [{ output: { exit_code: 0 }, source }]) + expect(spans.find((item) => item.attributes['openinference.span.kind'] === 'TOOL')?.status.code).toBe('ERROR') + }) + + it('keeps an unmatched call unknown', async () => { + const spans = await parseOutputs(variant, [{ pending: true }]) + const tool = spans.find((item) => item.attributes['openinference.span.kind'] === 'TOOL')! + expect(tool.status).toEqual({ code: 'UNSET' }) + expect(tool.attributes['output.value']).toBeUndefined() + }) + + it('reports corrected error and retry counts through the analysis consumers', async () => { + const successful = { output: 'Process exited with code 0\nOutput:\nerror: command failed ENOENT {"success":false}' } + const spans = await parseOutputs(variant, [successful, successful]) + const investigation = await runTraceInvestigation({ + spans, + harness: 'codex', + otlpOutPath: join(dir, `${variant}.otlp.jsonl`), + }) + const report = investigation.pipelines + expect(report.toolUse[0]).toMatchObject({ totalCalls: 2, errorRate: 0, retryRate: 0 }) + expect(report.failureClusters).toMatchObject({ totalFailures: 0, totalRuns: 1 }) + expect(report.failureFollowUps?.failures).toBe(0) + const execution = renderExecution(summarizeSpanExecution(spans)) + expect(execution).toContain('**Terminal outcomes:** 1 succeeded | 0 failed') + expect(execution).toContain('**Sessions with execution errors:** 0/1 (0.00%)') + + const failed = await runPipelines(await parseOutputs(variant, [ + { output: { exit_code: 1, output: 'success' } }, + successful, + { output: 'unknown' }, + ])) + expect(failed.toolUse[0]).toMatchObject({ totalCalls: 3, errorRate: 1 / 3, retryRate: 1 }) + + const unknownRetry = await runPipelines(await parseOutputs(variant, [ + { output: { exit_code: 1 } }, + { output: 'unknown' }, + ])) + expect(unknownRetry.failureFollowUps).toMatchObject({ + failures: 1, + followed: 1, + followUpSucceeded: 0, + items: [expect.objectContaining({ followUpSucceeded: null })], + }) + }) +}) diff --git a/tests/failure-followup.test.ts b/tests/failure-followup.test.ts index 6d33872..f4509d9 100644 --- a/tests/failure-followup.test.ts +++ b/tests/failure-followup.test.ts @@ -8,7 +8,7 @@ function toolCall( i: number, name: string, input: unknown, - status: 'OK' | 'ERROR' = 'OK', + status: 'OK' | 'ERROR' | 'UNSET' = 'OK', extra?: Record, ) { const startMs = 1_000 + i * 1000 @@ -40,6 +40,22 @@ const root = span({ }) describe('classifyFailureFollowUps', () => { + it('keeps an unknown follow-up outcome out of the success count', async () => { + const spans = [ + root, + toolCall(1, 'bash', { cmd: 'npm test' }, 'ERROR'), + toolCall(2, 'bash', { cmd: 'npm test' }, 'UNSET'), + ] + const result = await runPipelines(spans) + expect(result.failureFollowUps).toMatchObject({ + failures: 1, + followed: 1, + followUpSucceeded: 0, + items: [expect.objectContaining({ followUpSucceeded: null, kind: 'blind' })], + }) + expect(renderPipelines(result)).toContain('0/1 follow-ups succeeded') + }) + it('labels an identical re-send after a failure as blind', () => { const r = classifyFailureFollowUps([ root, diff --git a/tests/runtime-store.test.ts b/tests/runtime-store.test.ts index 14eec6f..bc2c0c7 100644 --- a/tests/runtime-store.test.ts +++ b/tests/runtime-store.test.ts @@ -23,6 +23,17 @@ function trace(traceId: string) { } describe('toRuntimeStore', () => { + it('preserves unknown source status instead of manufacturing runtime success', async () => { + const spans = trace('trace-a') + spans[1]!.status = { code: 'UNSET' } + const { store } = await toRuntimeStore(spans) + const runtimeSpans = await store.spans() + expect(runtimeSpans.find((item) => item.spanId === 'trace-a:root')?.status).toBe('ok') + const unknown = runtimeSpans.find((item) => item.spanId === 'trace-a:step-1') + expect(unknown).toBeDefined() + expect(unknown?.status).toBeUndefined() + }) + it('namespaces span and parent IDs so updates affect exactly one trace', async () => { const { store } = await toRuntimeStore([...trace('trace-a'), ...trace('trace-b')]) diff --git a/tests/upgrades.test.ts b/tests/upgrades.test.ts index 7062d33..b024d8b 100644 --- a/tests/upgrades.test.ts +++ b/tests/upgrades.test.ts @@ -376,7 +376,12 @@ describe('analyzeAdoption', () => { expect(r.skillRunFilesRead).toBe(2) }) - it('reports Codex skill invocation as unsupported while preserving catalog, file, and subagent evidence', async () => { + it.each([ + { label: 'successful read', output: '{"exit_code":0,"output":"# Simplify"}', status: 'OK', reads: 1 }, + { label: 'failed read', output: '{"exit_code":1,"output":"file not found"}', status: 'ERROR', reads: 0 }, + { label: 'unknown read outcome', output: 'Script completed', status: 'UNSET', reads: 0 }, + { label: 'running read', output: '{"session_id":123,"exit_code":null,"output":""}', status: 'UNSET', reads: 0 }, + ])('preserves Codex catalog and subagent evidence with $label', async ({ output, status, reads }) => { const path = join(dir, 'rollout-codex-skill-evidence.jsonl') const rows = [ { type: 'session_meta', timestamp: '2026-06-20T00:00:00Z', payload: { id: 'codex-skill-evidence', cwd: '/x' } }, @@ -396,13 +401,13 @@ describe('analyzeAdoption', () => { type: 'custom_tool_call', call_id: 'skill-file', name: 'exec', - input: 'const r = await tools.exec_command({cmd:"sed -n 1,80p /skills/simplify/SKILL.md"}); text(r.output)', + input: 'const r = await tools.exec_command({cmd:"sed -n 1,80p /skills/simplify/SKILL.md"}); text(r)', }, }, { type: 'response_item', timestamp: '2026-06-20T00:00:03Z', - payload: { type: 'custom_tool_call_output', call_id: 'skill-file', output: 'Script completed' }, + payload: { type: 'custom_tool_call_output', call_id: 'skill-file', output }, }, { type: 'response_item', @@ -411,13 +416,13 @@ describe('analyzeAdoption', () => { type: 'custom_tool_call', call_id: 'skill-catalog', name: 'exec', - input: 'const r = await tools.exec_command({cmd:"rg --files /skills/simplify/SKILL.md"}); text(r.output)', + input: 'const r = await tools.exec_command({cmd:"rg --files /skills/simplify/SKILL.md"}); text(r)', }, }, { type: 'response_item', timestamp: '2026-06-20T00:00:03.200Z', - payload: { type: 'custom_tool_call_output', call_id: 'skill-catalog', output: 'Script completed' }, + payload: { type: 'custom_tool_call_output', call_id: 'skill-catalog', output: '{"exit_code":0,"output":"/skills/simplify/SKILL.md"}' }, }, { type: 'event_msg', @@ -433,6 +438,8 @@ describe('analyzeAdoption', () => { writeFileSync(path, rows.map((row) => JSON.stringify(row)).join('\n')) const spans = await new CodexAdapter().parse(refFor(path, 'codex')) + const skillRead = spans.find((item) => String(item.attributes['input.value']).includes('sed -n')) + expect(skillRead?.status.code).toBe(status) const report = await analyzeAdoption(spans) const rendered = renderAdoption(report) @@ -440,14 +447,18 @@ describe('analyzeAdoption', () => { expect(report.skillTelemetryStatus).toBe('unsupported') expect(report.skillTelemetrySessions).toBe(0) expect(report.sessionsWithMaterializedSkills).toBe(1) - expect(report.sessionsWithSkillFileReference).toBe(1) - expect(report.skillDocumentReads.simplify).toBe(1) + expect(report.sessionsWithSkillFileReference).toBe(reads) + expect(report.skillDocumentReads).toEqual(reads === 1 ? { simplify: 1 } : {}) expect(report.totalSubagentSpawns).toBe(1) expect(report.subagentSpawns.reviewer).toBe(1) expect(rendered).toContain('Explicit skill invocation rate:** uncaptured/unsupported') expect(rendered).toContain('Materialized skill catalogs/instructions:** 1/1') - expect(rendered).toContain('Sessions with successful skill-document reads:** 1/1') - expect(rendered).toContain('Successful skill-document reads:** 1; inspection is not outcome evidence.') + expect(rendered).toContain(`Sessions with successful skill-document reads:** ${reads}/1`) + if (reads > 0) { + expect(rendered).toContain(`Successful skill-document reads:** ${reads}; inspection is not outcome evidence.`) + } else { + expect(rendered).not.toContain('- **Successful skill-document reads:**') + } expect(rendered).toContain('Subagent spawns observed:** 1') expect(rendered).toContain('Prompt, tools, MCP, hooks, and full agent profile:** not assessed') expect(rendered).not.toContain('Skill penetration')