diff --git a/docs/trace-analysts.md b/docs/trace-analysts.md index 102ff6d..bb3794e 100644 --- a/docs/trace-analysts.md +++ b/docs/trace-analysts.md @@ -72,6 +72,27 @@ The runtime span projection leaves its optional status absent for `UNSET` spans. Use the execution report for terminal run outcomes. The older runtime store's required run status cannot represent unknown and is not completion evidence. +## Model-assisted citations + +agent-eval's evidence gate accepts a model finding only when each cited span exists and each excerpt appears in that span. +Before the gate runs, `traces` normalizes the citations of the built-in `--llm` analysts against the analyzed spans. + +- An excerpt copied from a `searchTrace` hit carries JSON escapes such as `\"` and `\n`. + It is replaced by its decoded form only when the cited span contains that decoded text. +- A citation of a readable harness ID, such as `tool:` or the Codex session ID, is replaced by the hex OTLP span ID. + The readable ID must name exactly one span through its `traces..source_*` attributes. +- A subject from another analyst's vocabulary is removed, because the subject is optional. + The claim and its citations still go through the gate. + +The gate itself is unchanged and checks every rewritten finding. +A fabricated excerpt, an unknown ID, or an ambiguous ID remains rejected. +Normalization never adds a citation, so a failure-mode finding still needs two distinct spans. +A subject outside the subject grammar is rejected before normalization runs. +Wrapped analysts report a version ending in `+traces-citations-1`. + +A custom analyst built with `createTraceAnalyst` gets the same behavior through `normalizeAnalystCitations(definitions, spans)`. +Pass the spans that the analyst's trace store was written from. + ## External engines External engines are optional tools that you install separately. diff --git a/src/analyst-citations.ts b/src/analyst-citations.ts new file mode 100644 index 0000000..123f7ed --- /dev/null +++ b/src/analyst-citations.ts @@ -0,0 +1,254 @@ +/** + * Citation normalization for model-backed trace analysts. + * + * agent-eval's evidence gate accepts a finding only when every citation + * resolves to a stored span and every excerpt is an exact substring of that + * span's decoded content. Two artifacts of how `traces` exports spans make the + * engine's own honest citations fail that check: + * + * - `searchTrace` returns raw OTLP-JSONL text, so an excerpt copied from a hit + * carries JSON escapes (`\"`, `\n`) that the decoded attribute does not. + * - Adapters rewrite readable harness IDs (`tool:`, a session UUID) + * to fixed-width hex OTLP IDs and keep the readable form only in + * `traces..source_trace_id` / `source_span_id` attributes, so a + * citation of the readable ID names no stored span. + * + * A third rejection is a subject from another kind's vocabulary (for example a + * `tool-doc:` locus on the cluster-only failure-mode kind). The subject is an + * optional label, and the finding schema tells the model to omit it rather + * than guess, so a subject the kind cannot hold is removed. + * + * Protected invariant: normalization never lets absent evidence pass. A + * citation is rewritten only to a form proven present in the span it cites: a + * readable ID must name exactly one span, and a decoded excerpt must be a + * substring of that span's own attributes or status message. Anything else is + * left untouched, and agent-eval's gate re-checks every rewritten row + * unchanged. Normalization never adds a citation, so it cannot satisfy a + * kind's minimum citation count on its own. + * + * Temporary: the agent-eval gate issue (agent-eval#TBD) makes this module + * unnecessary. Delete it when traces adopts a release whose gate accepts + * JSON-escaped excerpts, resolves the readable source-ID aliases above, and + * treats an out-of-kind optional subject as omitted. + */ + +import { + KIND_EXPECTED_SUBJECTS, + parseFindingSubject, + type AnalystContext, + type RawAnalystFinding, + type TraceAnalystDefinition, +} from '@tangle-network/agent-eval/analyst' +import { spanEvidenceUri } from './external-analysis-validation.js' +import type { OtlpSpan } from './otlp.js' + +/** + * Appended to each wrapped definition's version. agent-eval records a + * `postProcess` hook as `version-bound`, so the version names this behavior. + * Bump it whenever the normalization rules change. + */ +export const CITATION_NORMALIZATION_VERSION = 'traces-citations-1' + +const SOURCE_TRACE_ID = /^traces\.([a-z0-9_-]+)\.source_trace_id$/ +const SPAN_URI_PREFIX = 'trace://' +const SPAN_URI_SEPARATOR = '/span/' +// JSON string escapes only. Anything else stays literal, so a truncated or +// already-decoded excerpt cannot turn into a different string. +const JSON_ESCAPE = /\\(?:u([0-9a-fA-F]{4})|(["\\/bfnrt]))/g +const SIMPLE_ESCAPES: Readonly> = { + '"': '"', + '\\': '\\', + '/': '/', + b: '\b', + f: '\f', + n: '\n', + r: '\r', + t: '\t', +} + +interface IndexedSpan { + readonly traceId: string + readonly spanId: string + /** The values the gate searches for an excerpt: attributes and status message. */ + readonly content: readonly unknown[] +} + +interface CitationIndex { + readonly traces: ReadonlyMap> + /** Readable trace ID → hex trace IDs. */ + readonly readableTraces: ReadonlyMap> + /** Hex trace ID → readable span ID → spans carrying that alias. */ + readonly readableSpans: ReadonlyMap> +} + +/** + * Wrap trace-analyst definitions so each submitted finding is normalized + * against `spans` before agent-eval's evidence gate runs. + * + * Pass the same spans the analysts' trace store was written from. A + * definition's own `postProcess` runs after normalization and keeps the final + * say over the row. + */ +export function normalizeAnalystCitations( + definitions: readonly TraceAnalystDefinition[], + spans: readonly OtlpSpan[], +): TraceAnalystDefinition[] { + const index = indexSpans(spans) + return definitions.map((definition) => { + const own = definition.postProcess + return { + ...definition, + version: `${definition.version}+${CITATION_NORMALIZATION_VERSION}`, + postProcess: (row: RawAnalystFinding, context: AnalystContext) => { + const normalized = normalizeFinding(row, definition.id, index, context) + return own ? own(normalized, context) : normalized + }, + } + }) +} + +function normalizeFinding( + row: RawAnalystFinding, + analystId: string, + index: CitationIndex, + context: AnalystContext, +): RawAnalystFinding { + const evidence = row.evidence.map((citation) => { + const span = resolveCitation(citation.uri, index) + if (!span) return citation + const uri = spanEvidenceUri(span.traceId, span.spanId) + if (uri !== citation.uri) { + context.log?.('finding citation normalized: readable id', { + analyst_id: analystId, + from: citation.uri, + to: uri, + }) + } + const excerpt = citation.excerpt === undefined + ? undefined + : normalizeExcerpt(citation.excerpt, span) + if (excerpt !== citation.excerpt) { + context.log?.('finding citation normalized: json-escaped excerpt', { + analyst_id: analystId, + uri, + }) + } + return excerpt === undefined ? { uri } : { uri, excerpt } + }) + const normalized: RawAnalystFinding = { ...row, evidence } + const allowed = KIND_EXPECTED_SUBJECTS[analystId] + if (row.subject === undefined || !allowed) return normalized + const subject = parseFindingSubject(row.subject) + // A grammar-invalid subject never reaches postProcess: the schema rejects + // the row first. Only a valid subject from another kind's vocabulary lands here. + if (!subject || allowed.includes(subject.kind)) return normalized + context.log?.('finding subject omitted: not valid for analyst', { + analyst_id: analystId, + subject: row.subject, + allowed, + }) + const { subject: _omitted, ...withoutSubject } = normalized + return withoutSubject +} + +/** The one span a citation names, by hex or readable ID, or null when absent or ambiguous. */ +function resolveCitation(uri: string, index: CitationIndex): IndexedSpan | null { + const parsed = parseCitation(uri) + if (!parsed) return null + const traceIds = parsed.traceId === undefined + ? [...index.traces.keys()] + : index.traces.has(parsed.traceId) + ? [parsed.traceId] + : [...(index.readableTraces.get(parsed.traceId) ?? [])] + const matches = new Set() + for (const traceId of traceIds) { + const direct = index.traces.get(traceId)?.get(parsed.spanId) + if (direct) matches.add(direct) + for (const span of index.readableSpans.get(traceId)?.get(parsed.spanId) ?? []) matches.add(span) + } + if (matches.size !== 1) return null + const [span] = matches + return span ?? null +} + +/** + * Split a span citation at its first `/span/`, so a readable span ID that + * contains `/` still resolves. A bare ID without a scheme names a span in any + * trace, and resolves only when exactly one span carries it. + */ +function parseCitation(uri: string): { traceId?: string; spanId: string } | null { + const trimmed = uri.trim() + if (trimmed.startsWith(SPAN_URI_PREFIX)) { + const rest = trimmed.slice(SPAN_URI_PREFIX.length) + const separator = rest.indexOf(SPAN_URI_SEPARATOR) + if (separator <= 0) return null + const traceId = decodeComponent(rest.slice(0, separator)) + const spanId = decodeComponent(rest.slice(separator + SPAN_URI_SEPARATOR.length)) + return traceId && spanId ? { traceId, spanId } : null + } + if (trimmed.includes('://')) return null + return trimmed ? { spanId: trimmed } : null +} + +function decodeComponent(value: string): string { + try { + return decodeURIComponent(value) + } catch { + return value + } +} + +/** Keep an excerpt the span already contains; otherwise decode it only when the decoded text is in the span. */ +function normalizeExcerpt(excerpt: string, span: IndexedSpan): string { + if (containsText(span.content, excerpt)) return excerpt + const decoded = decodeJsonEscapes(excerpt) + return decoded !== excerpt && containsText(span.content, decoded) ? decoded : excerpt +} + +function decodeJsonEscapes(text: string): string { + return text.replace(JSON_ESCAPE, (_match, unicode: string | undefined, simple: string | undefined) => + unicode !== undefined ? String.fromCharCode(Number.parseInt(unicode, 16)) : SIMPLE_ESCAPES[simple!]!, + ) +} + +/** Mirrors the gate's search: string values at any depth, never keys. */ +function containsText(value: unknown, expected: string, depth = 0): boolean { + if (!expected || depth > 20) return false + if (typeof value === 'string') return value.includes(expected) + if (Array.isArray(value)) return value.some((entry) => containsText(entry, expected, depth + 1)) + if (typeof value === 'object' && value !== null) { + return Object.values(value).some((entry) => containsText(entry, expected, depth + 1)) + } + return false +} + +function indexSpans(spans: readonly OtlpSpan[]): CitationIndex { + const traces = new Map>() + const readableTraces = new Map>() + const readableSpans = new Map>() + for (const item of spans) { + const span: IndexedSpan = { + traceId: item.trace_id, + spanId: item.span_id, + content: [item.attributes, item.status.message], + } + let byId = traces.get(item.trace_id) + if (!byId) traces.set(item.trace_id, byId = new Map()) + byId.set(item.span_id, span) + for (const [key, value] of Object.entries(item.attributes)) { + const harness = SOURCE_TRACE_ID.exec(key)?.[1] + if (!harness || typeof value !== 'string') continue + let hexTraces = readableTraces.get(value) + if (!hexTraces) readableTraces.set(value, hexTraces = new Set()) + hexTraces.add(item.trace_id) + const readableSpanId = item.attributes[`traces.${harness}.source_span_id`] + if (typeof readableSpanId !== 'string') continue + let aliases = readableSpans.get(item.trace_id) + if (!aliases) readableSpans.set(item.trace_id, aliases = new Map()) + const existing = aliases.get(readableSpanId) + if (existing) existing.push(span) + else aliases.set(readableSpanId, [span]) + } + } + return { traces, readableTraces, readableSpans } +} diff --git a/src/analyze.ts b/src/analyze.ts index 41ee982..24aca5c 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -18,10 +18,12 @@ import { type AnalystRegistry, type AnalystRunSummary, buildDefaultAnalystRegistry, + DEFAULT_TRACE_ANALYST_KINDS, type TraceAnalysisEngine, type TraceAnalystDefinition, } from '@tangle-network/agent-eval/analyst' import { OtlpFileTraceStore } from '@tangle-network/agent-eval/traces' +import { normalizeAnalystCitations } from './analyst-citations.js' import { summarizeSpanExecution } from './execution.js' import type { OtlpSpan } from './otlp.js' import { writeOtlpFile } from './otlp.js' @@ -134,7 +136,9 @@ export async function analyzeSpans(spans: readonly OtlpSpan[], opts: AnalyzeOpti await agStore.ensureIndexed() const agRegistry = opts.agenticRegistry ?? buildDefaultAnalystRegistry({ engine: opts.engine!, - ...(opts.agenticKinds ? { definitions: opts.agenticKinds } : {}), + // The store above is written from these spans, so citations are + // normalized against exactly the content the evidence gate re-reads. + definitions: normalizeAnalystCitations(opts.agenticKinds ?? DEFAULT_TRACE_ANALYST_KINDS, spans), includeBehavioral: false, registry: { log: opts.log }, }) diff --git a/src/index.ts b/src/index.ts index 58bc6a0..7fd7313 100644 --- a/src/index.ts +++ b/src/index.ts @@ -88,6 +88,7 @@ export * from './adoption.js' // analyzeAdoption() — skill + subagent metrics export * from './agentic-routing.js' // planTraceAgenticRoute(): deterministic LLM analyst routing export * from './runtime-store.js' // toRuntimeStore() — feed agent-eval pipelines export * from './analyze.js' // analyzeSpans({ registry? }) — run YOUR analysts +export * from './analyst-citations.js' // normalizeAnalystCitations() — model citations the evidence gate can check export * from './execution.js' // shared execution accounting over normalized spans export * from './evidence.js' // policy-evidence JSONL for downstream miners export * from './session-index.js' // collectSessionIndex() — reusable session catalog diff --git a/tests/analyst-citations.test.ts b/tests/analyst-citations.test.ts new file mode 100644 index 0000000..69f3eaa --- /dev/null +++ b/tests/analyst-citations.test.ts @@ -0,0 +1,347 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { + type AnalystFinding, + createTraceAnalyst, + FAILURE_MODE_KIND_SPEC, + type RawAnalystEvidence, + type RawAnalystFinding, + type TraceAnalysisEngine, + type TraceAnalysisEngineRequest, + type TraceAnalystDefinition, +} from '@tangle-network/agent-eval/analyst' +import { OtlpFileTraceStore } from '@tangle-network/agent-eval/traces' +import { CodexAdapter } from '../src/adapters/codex.js' +import { CITATION_NORMALIZATION_VERSION, normalizeAnalystCitations } from '../src/analyst-citations.js' +import { runTraceInvestigation } from '../src/improvement.js' +import { type OtlpSpan, writeOtlpFile } from '../src/otlp.js' + +const dir = mkdtempSync(join(tmpdir(), 'traces-citations-')) +afterAll(() => rmSync(dir, { recursive: true, force: true })) + +const SESSION = 'citation-session' +const LINT_OUTPUT = 'Process exited with code 1\nOutput:\nerror: "fixture.lock" is stale\nrun the setup step again' +const DECODED_EXCERPT = 'Output:\nerror: "fixture.lock" is stale' +// What a model copies from a searchTrace hit: the raw OTLP-JSONL text, escapes included. +const ESCAPED_EXCERPT = 'Output:\\nerror: \\"fixture.lock\\" is stale' + +function codexSession(sessionId: string): Promise { + const path = join(dir, `${sessionId}.jsonl`) + const events: Record[] = [ + { type: 'session_meta', payload: { id: sessionId, cwd: '/fixture' } }, + { type: 'event_msg', payload: { type: 'task_started', turn_id: 'turn-1' } }, + { type: 'response_item', payload: { type: 'function_call', call_id: 'call-lint', name: 'exec_command', arguments: '{"cmd":"pnpm lint"}' } }, + { type: 'response_item', payload: { type: 'function_call_output', call_id: 'call-lint', output: LINT_OUTPUT } }, + { type: 'response_item', payload: { type: 'function_call', call_id: 'call-setup', name: 'exec_command', arguments: '{"cmd":"pnpm setup"}' } }, + { type: 'response_item', payload: { type: 'function_call_output', call_id: 'call-setup', output: 'Process exited with code 0\nOutput:\nsetup complete: 3 packages linked' } }, + { type: 'event_msg', payload: { type: 'task_complete', turn_id: 'turn-1' } }, + ] + writeFileSync(path, events.map((event, index) => JSON.stringify({ + ...event, + timestamp: new Date(Date.UTC(2026, 8, 8, 0, 0, index)).toISOString(), + })).join('\n')) + return new CodexAdapter().parse({ harness: 'codex', sessionId, path, cwd: null, mtimeMs: 0 }) +} + +async function traceStore(spans: readonly OtlpSpan[], name: string): Promise { + const store = new OtlpFileTraceStore({ path: await writeOtlpFile(spans, join(dir, `${name}.otlp.jsonl`)) }) + await store.ensureIndexed() + return store +} + +/** What the stub engine learned from the real trace tools, as a model would. */ +interface Citations { + readonly traceId: string + readonly lint: string + readonly setup: string + readonly readableLint: string + readonly readableSetup: string + readonly searchText: string +} + +type Tools = TraceAnalysisEngineRequest['tools'] + +async function callTool(tools: Tools, name: string, args: unknown): Promise { + const tool = tools.find((entry) => entry.name === name) + if (!tool) throw new Error(`tool ${name} was not supplied`) + return await tool.handler(args) as T +} + +interface SearchResult { + hits: Array<{ span_id: string; matched_text: string; context_before: string; context_after: string }> +} + +async function readCitations(tools: Tools): Promise { + const overview = await callTool<{ sample_trace_ids: string[] }>(tools, 'getDatasetOverview', {}) + const traceId = overview.sample_trace_ids[0]! + const lintHit = (await callTool(tools, 'searchTrace', { trace_id: traceId, regex_pattern: 'is stale' })).hits[0]! + const setupHit = (await callTool(tools, 'searchTrace', { trace_id: traceId, regex_pattern: 'setup complete' })).hits[0]! + const viewed = await callTool<{ spans: Array<{ span_id: string; attributes: Record }> }>( + tools, + 'viewSpans', + { trace_id: traceId, span_ids: [lintHit.span_id, setupHit.span_id] }, + ) + const readable = (spanId: string) => String( + viewed.spans.find((span) => span.span_id === spanId)?.attributes['traces.codex.source_span_id'], + ) + return { + traceId, + lint: lintHit.span_id, + setup: setupHit.span_id, + readableLint: readable(lintHit.span_id), + readableSetup: readable(setupHit.span_id), + searchText: lintHit.context_before + lintHit.matched_text + lintHit.context_after, + } +} + +type FindingBuilder = (citations: Citations) => Record + +/** Submits fixed findings built from real tool reads; no model is called. */ +function stubEngine(build: FindingBuilder, seen: Citations[] = []): TraceAnalysisEngine { + return { + id: 'citation-stub', + description: 'Submits fixed findings built from real trace-tool reads.', + model: 'stub-model', + version: '1.0.0', + executionConfig: {}, + async analyze(request) { + if (request.analystId !== FAILURE_MODE_KIND_SPEC.id) { + return { answer: 'no findings', findings: [], trajectory: [], modelCalls: 0, toolCalls: 0, runtime: {} } + } + const citations = await readCitations(request.tools) + seen.push(citations) + return { + answer: 'stub answer', + findings: [build(citations) as RawAnalystFinding], + trajectory: [], + modelCalls: 0, + toolCalls: 4, + runtime: {}, + } + }, + } +} + +function spanUri(traceId: string, spanId: string): string { + return `trace://${encodeURIComponent(traceId)}/span/${encodeURIComponent(spanId)}` +} + +function finding(claim: string, evidence: RawAnalystEvidence[], extra: Record = {}) { + return { severity: 'high', claim, confidence: 0.9, evidence, ...extra } +} + +interface GateOutcome { + readonly accepted: readonly AnalystFinding[] + readonly rejections: readonly string[] + readonly citations: Citations +} + +async function gate( + definition: TraceAnalystDefinition, + store: OtlpFileTraceStore, + build: FindingBuilder, +): Promise { + const rejections: string[] = [] + const seen: Citations[] = [] + const analyst = createTraceAnalyst(definition, { engine: stubEngine(build, seen) }) + const accepted = await analyst.analyze(store, { + runId: 'citations', + correlationId: 'citations', + log: (message, fields) => { + if (message.startsWith('finding rejected')) rejections.push(String(fields?.reason ?? message)) + }, + }) + return { accepted, rejections, citations: seen[0]! } +} + +describe('analyst citation normalization', () => { + let spans: OtlpSpan[] + let store: OtlpFileTraceStore + let normalized: TraceAnalystDefinition + + beforeAll(async () => { + spans = await codexSession(SESSION) + store = await traceStore(spans, 'single') + normalized = normalizeAnalystCitations([FAILURE_MODE_KIND_SPEC], spans)[0]! + }) + + // Each shape cites two real spans: failure-mode needs the error span and the + // span where work resumed. Only the citation form differs from the gate's. + const realEvidence: Array<{ shape: string; before: string; build: FindingBuilder }> = [ + { + shape: 'escaped excerpt copied from a search hit', + before: 'excerpt is not present in the cited span content', + build: (c) => finding('escaped excerpt', [ + { uri: spanUri(c.traceId, c.lint), excerpt: ESCAPED_EXCERPT }, + { uri: spanUri(c.traceId, c.setup) }, + ]), + }, + { + shape: 'readable span id in the hex trace', + before: 'trace span does not exist', + build: (c) => finding('readable span id', [ + { uri: spanUri(c.traceId, c.lint) }, + { uri: spanUri(c.traceId, c.readableSetup) }, + ]), + }, + { + shape: 'readable trace and span ids', + before: 'trace span does not exist', + build: (c) => finding('readable trace and span ids', [ + { uri: `trace://${SESSION}/span/${c.readableLint}`, excerpt: DECODED_EXCERPT }, + { uri: spanUri(c.traceId, c.setup) }, + ]), + }, + { + shape: 'bare readable id', + before: 'citation is not a supplied finding or trace span', + build: (c) => finding('bare readable id', [ + { uri: c.readableLint }, + { uri: spanUri(c.traceId, c.setup) }, + ]), + }, + { + shape: 'subject from another kind', + before: 'finding rejected: subject is not valid for analyst', + build: (c) => finding('subject from another kind', [ + { uri: spanUri(c.traceId, c.lint) }, + { uri: spanUri(c.traceId, c.setup) }, + ], { subject: 'tool-doc:exec_command' }), + }, + ] + + it('reads the escaped excerpt from a real search hit, and the readable ids from the spans', async () => { + const { citations } = await gate(normalized, store, realEvidence[0]!.build) + expect(citations.searchText).toContain(ESCAPED_EXCERPT) + expect(citations.searchText).not.toContain(DECODED_EXCERPT) + expect(citations.readableLint).toBe('tool:call-lint') + expect(citations.readableSetup).toBe('tool:call-setup') + }) + + it.each(realEvidence)('gate rejects the unnormalized $shape', async ({ before, build }) => { + const outcome = await gate(FAILURE_MODE_KIND_SPEC, store, build) + expect(outcome.accepted).toEqual([]) + expect(outcome.rejections).toEqual([before]) + }) + + it.each(realEvidence)('gate accepts the normalized $shape', async ({ build }) => { + const outcome = await gate(normalized, store, build) + expect(outcome.rejections).toEqual([]) + expect(outcome.accepted).toHaveLength(1) + const [accepted] = outcome.accepted + const { citations: c } = outcome + expect(accepted!.evidence_refs.map((ref) => ref.uri).sort()) + .toEqual([spanUri(c.traceId, c.lint), spanUri(c.traceId, c.setup)].sort()) + expect(accepted!.subject).toBeUndefined() + expect(accepted!.metadata?.definition_version) + .toBe(`${FAILURE_MODE_KIND_SPEC.version}+${CITATION_NORMALIZATION_VERSION}`) + }) + + it('rewrites an escaped excerpt to the decoded text present in the cited span', async () => { + const { accepted } = await gate(normalized, store, realEvidence[0]!.build) + expect(accepted[0]!.evidence_refs.find((ref) => ref.excerpt !== undefined)?.excerpt).toBe(DECODED_EXCERPT) + }) + + const stillRejected: Array<{ shape: string; reason: string; build: FindingBuilder }> = [ + { + shape: 'fabricated escaped excerpt', + reason: 'excerpt is not present in the cited span content', + build: (c) => finding('fabricated excerpt', [ + { uri: spanUri(c.traceId, c.lint), excerpt: 'Output:\\nerror: \\"fixture.lock\\" is fresh' }, + { uri: spanUri(c.traceId, c.setup) }, + ]), + }, + { + shape: 'real excerpt cited on a span that does not contain it', + reason: 'excerpt is not present in the cited span content', + build: (c) => finding('excerpt on the wrong span', [ + { uri: spanUri(c.traceId, c.setup), excerpt: ESCAPED_EXCERPT }, + { uri: spanUri(c.traceId, c.lint) }, + ]), + }, + { + shape: 'fabricated readable id', + reason: 'trace span does not exist', + build: (c) => finding('fabricated readable id', [ + { uri: spanUri(c.traceId, c.lint) }, + { uri: spanUri(c.traceId, 'tool:call-missing') }, + ]), + }, + { + // failure-mode requires the error span and the recovery span. + shape: 'single citation', + reason: 'finding rejected: insufficient evidence citations', + build: (c) => finding('single citation', [{ uri: spanUri(c.traceId, c.lint) }]), + }, + { + // Normalization collapses an alias onto its span; it cannot add a citation. + shape: 'readable alias of a span already cited', + reason: 'finding rejected: insufficient evidence citations', + build: (c) => finding('alias of a cited span', [ + { uri: spanUri(c.traceId, c.lint) }, + { uri: c.readableLint }, + ]), + }, + { + // The schema rejects this before postProcess runs; see the agent-eval gate issue. + shape: 'subject outside the grammar', + reason: 'finding rejected: schema failure', + build: (c) => finding('subject outside the grammar', [ + { uri: spanUri(c.traceId, c.lint) }, + { uri: spanUri(c.traceId, c.setup) }, + ], { subject: 'Probe Upper' }), + }, + ] + + it.each(stillRejected)('gate still rejects a normalized $shape', async ({ reason, build }) => { + const outcome = await gate(normalized, store, build) + expect(outcome.accepted).toEqual([]) + expect(outcome.rejections).toEqual([reason]) + }) + + it('leaves a readable id untouched when it names spans in two traces', async () => { + const both = [...await codexSession('citation-a'), ...await codexSession('citation-b')] + const twoTraces = await traceStore(both, 'two-traces') + const definition = normalizeAnalystCitations([FAILURE_MODE_KIND_SPEC], both)[0]! + const setupOf = (sessionId: string) => both.find((span) => + span.attributes['traces.codex.source_trace_id'] === sessionId + && span.attributes['traces.codex.source_span_id'] === 'tool:call-setup')! + + const ambiguous = await gate(definition, twoTraces, () => { + const setup = setupOf('citation-a') + return finding('ambiguous bare id', [{ uri: 'tool:call-lint' }, { uri: spanUri(setup.trace_id, setup.span_id) }]) + }) + expect(ambiguous.accepted).toEqual([]) + expect(ambiguous.rejections).toEqual(['citation is not a supplied finding or trace span']) + + const scoped = await gate(definition, twoTraces, () => { + const setup = setupOf('citation-a') + return finding('scoped readable id', [ + { uri: 'trace://citation-a/span/tool:call-lint' }, + { uri: spanUri(setup.trace_id, setup.span_id) }, + ]) + }) + expect(scoped.rejections).toEqual([]) + const lint = both.find((span) => + span.attributes['traces.codex.source_trace_id'] === 'citation-a' + && span.attributes['traces.codex.source_span_id'] === 'tool:call-lint')! + expect(scoped.accepted[0]!.evidence_refs[0]!.uri).toBe(spanUri(lint.trace_id, lint.span_id)) + }) + + it('normalizes the built-in kinds that trace investigations run', async () => { + const result = await runTraceInvestigation({ + spans, + harness: 'codex', + engine: stubEngine((c) => finding('investigation citation', [ + { uri: spanUri(c.traceId, c.lint), excerpt: ESCAPED_EXCERPT }, + { uri: spanUri(c.traceId, c.readableSetup) }, + ])), + generatedAt: '2026-09-08T00:00:00.000Z', + }) + const accepted = result.analystResult.findings.filter((entry) => entry.analyst_id === FAILURE_MODE_KIND_SPEC.id) + expect(accepted.map((entry) => entry.claim)).toEqual(['investigation citation']) + expect(accepted[0]!.evidence_refs.map((ref) => ref.excerpt)).toEqual([DECODED_EXCERPT, undefined]) + }) +})