diff --git a/src/adapters/codex-format.ts b/src/adapters/codex-format.ts index ee7f328..51ce63c 100644 --- a/src/adapters/codex-format.ts +++ b/src/adapters/codex-format.ts @@ -44,8 +44,24 @@ export interface CodexLine { author?: string recipient?: string namespace?: string - /** `user_message` event text. */ + /** `user_message` event text, and the summary on a `compacted` record. */ message?: unknown + /** + * Conversation history a `compacted` record retained, in response-item + * shape. Codex writes the pre-compaction turns here and nowhere else, so + * this is the only surviving copy of what the human typed before the + * context was replaced. + */ + replacement_history?: ReadonlyArray<{ + type?: string + id?: string + role?: string + content?: unknown + }> + window_id?: string + previous_window_id?: string + first_window_id?: string + window_number?: number item?: { type?: string id?: string diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index 6edd98d..66ce365 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -62,6 +62,16 @@ import { isCodexTaskBoundary, resolveCodexParentTask, } from './codex-task-scope.js' +import { + INHERITED_SOURCE_ATTR, + INHERITED_SPAN_ATTR, + INHERITED_SPAN_COUNT_ATTR, + INHERITED_SPANS_OMITTED_ATTR, + type InheritedSpanSource, + isInheritedSpan, + SYNTHESIZED_SOURCE_ATTR, + SYNTHESIZED_SPAN_ATTR, +} from './provenance.js' import { INNER_TOOL_CALL_LEVEL, recordToolOutput, TOOL_CALL_LEVEL_ATTR, toolIoAttributes } from './tool-io.js' export { CodexTaskScopeError } from './codex-task-scope.js' @@ -69,10 +79,41 @@ export { CodexTaskScopeError } from './codex-task-scope.js' const SERVICE = 'codex' const SESSION_HEAD_LINES = 40 +/** + * Per-session ceiling on inherited spans. The prefix of a fork and the retained + * history of every `compacted` record are unbounded in a long rollout, and this + * adapter parses under a bounded heap. What the cap drops is counted on the root + * in `traces.session.inherited_spans_omitted`, never silently discarded. + */ +const MAX_INHERITED_SPANS = 200 + const CODEX_SOURCE_TRACE_ID = 'traces.codex.source_trace_id' const CODEX_SOURCE_SPAN_ID = 'traces.codex.source_span_id' const CODEX_SOURCE_PARENT_SPAN_ID = 'traces.codex.source_parent_span_id' +/** + * The harness's own cumulative token counter, carried verbatim. + * + * Codex reports `token_count.info.total_token_usage` beside the per-turn + * `last_token_usage` delta. The two are different numbers and neither derives + * the other: summing the deltas misses whatever the harness counted outside the + * recorded turns, and summing the cumulative snapshots multiplies the session + * total by the number of events. The last snapshot IS the session total, so it + * is copied onto the root span and never recomputed here. + */ +const SESSION_TOTAL_TOKENS = 'traces.session.total_tokens' +const SESSION_TOTAL_INPUT_TOKENS = 'traces.session.total_input_tokens' +const SESSION_TOTAL_OUTPUT_TOKENS = 'traces.session.total_output_tokens' +const SESSION_TOTAL_REASONING_TOKENS = 'traces.session.total_reasoning_tokens' +const SESSION_TOTAL_CACHED_INPUT_TOKENS = 'traces.session.total_cached_input_tokens' +const SESSION_TOTAL_TOKENS_SOURCE = 'traces.session.total_tokens_source' +const CODEX_TOTAL_TOKENS_SOURCE = 'codex.token_count.info.total_token_usage' + +/** Carry a reported counter only when it is a usable non-negative number. */ +function reportedCount(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined +} + /** Convert Codex's readable span identities to fixed-width OTLP wire IDs. */ function normalizeCodexIds(spans: OtlpSpan[]): void { for (const item of spans) { @@ -852,7 +893,11 @@ export class CodexAdapter implements HarnessTraceAdapter { let step = 0 let lastLlm = rootId let sawUserTurn = false + let sawInheritedTurn = false + let inheritedSpansEmitted = 0 + let inheritedSpansOmitted = 0 let lastCumulativeTokenUsage: string | undefined + let sessionTotalUsage: CodexTokenUsage | undefined let lastTimestamp: string | undefined const awaitingModel = model ? [] : [root] const toolWindows = new Map() @@ -868,6 +913,7 @@ export class CodexAdapter implements HarnessTraceAdapter { const unpairedUserItems: UserTurnCandidate[] = [] const unpairedUserEvents: UserTurnCandidate[] = [] const tasksWithUserEvents = new Set() + const inheritedTurnKeys = new Set() /** * Record one turn Codex reports as submitted input. Codex reports these * turns and never its own context blocks: the legacy `user_message` event @@ -908,6 +954,143 @@ export class CodexAdapter implements HarnessTraceAdapter { unpairedUserEvents.push({ span: turnSpan, key, task: taskIndex }) step += 1 } + /** Reserve a slot for one inherited span, counting what the cap turns away. */ + const claimInheritedSpan = (): boolean => { + if (inheritedSpansEmitted >= MAX_INHERITED_SPANS) { + inheritedSpansOmitted += 1 + return false + } + inheritedSpansEmitted += 1 + return true + } + /** + * A turn this session carries but did not receive: the prefix a fork copies + * from its parent, and the history a `compacted` record retains. Codex + * rewrites both into the child's rollout, and the task-scope walk used to + * drop them, so a forked child's human context reached no span at all and + * "what did the human ask for?" had no answer in the trace. + * + * The span is a normal `user.prompt` with its real actor, plus + * `traces.session.inherited` — a reader that wants the human's words finds + * them by name, and a count of THIS scope's turns excludes them by flag. + * + * Deduplicated on the turn text, because one retained turn is rewritten + * into every later `compacted` record. The key is bounded (length plus a + * text prefix) so a long session cannot grow the key set without bound. + */ + const recordInheritedTurn = ( + raw: string, + ts: string, + contentSource: SourceReferences, + source: InheritedSpanSource, + blocks?: readonly string[], + ): void => { + const prompt = capText(raw) + if (!prompt) return + const key = userTurnKey(raw) + const dedupKey = `${key.length}:${key.slice(0, 256)}` + if (inheritedTurnKeys.has(dedupKey)) return + if (!claimInheritedSpan()) return + inheritedTurnKeys.add(dedupKey) + const actor = codexActor({ text: prompt, blocks, isFirstUserTurn: !sawInheritedTurn }) + sawInheritedTurn = true + const turnSpan = userPromptSpan({ + traceId, + spanId: `inherited:${step}:user`, + parentSpanId: rootId, + startTime: ts, + content: prompt, + contentSource, + service: SERVICE, + agent: SERVICE, + step, + actor, + }) + turnSpan.attributes[INHERITED_SPAN_ATTR] = true + turnSpan.attributes[INHERITED_SOURCE_ATTR] = source + spans.push(turnSpan) + step += 1 + } + /** + * Records the parsed scope inherited rather than produced. Two shapes reach + * here: any line before the selected task boundary (the fork prefix), and a + * `compacted` record anywhere in the file. A compacted record carries the + * summary Codex replaced the context with, plus the history it retained. + */ + const recordInheritedContext = (l: CodexLine, ts: string): void => { + if (l.type === 'compacted') { + const payload = l.payload + if (!payload) return + const summary = capText(typeof payload.message === 'string' ? payload.message : '') + if (summary && claimInheritedSpan()) { + spans.push(span({ + traceId, + spanId: `inherited:${step}:compacted`, + parentSpanId: rootId, + name: 'session.compacted', + kind: 'CHAIN', + startTime: ts, + service: SERVICE, + agent: SERVICE, + step, + content: summary, + contentSource: textSources(payload, 'message'), + extra: { + [INHERITED_SPAN_ATTR]: true, + [INHERITED_SOURCE_ATTR]: 'compacted' satisfies InheritedSpanSource, + ...(typeof payload.window_number === 'number' ? { 'traces.codex.compaction_window_number': payload.window_number } : {}), + ...(payload.window_id ? { 'traces.codex.compaction_window_id': payload.window_id } : {}), + ...(payload.previous_window_id ? { 'traces.codex.compaction_previous_window_id': payload.previous_window_id } : {}), + }, + })) + step += 1 + } + for (const item of payload.replacement_history ?? []) { + if (!item || typeof item !== 'object') continue + if (item.type !== 'message' || item.role !== 'user') continue + recordInheritedTurn( + contentToString(item.content), + ts, + textSources(item, 'content'), + 'compacted', + contentTextBlocks(item.content), + ) + } + return + } + if (l.type === 'response_item' && l.payload?.type === 'message' && l.payload.role === 'user') { + recordInheritedTurn( + contentToString(l.payload.content), + ts, + textSources(l.payload, 'content'), + 'pre-task-prefix', + contentTextBlocks(l.payload.content), + ) + return + } + if (l.type === 'event_msg' && l.payload?.type === 'user_message') { + recordInheritedTurn( + typeof l.payload.message === 'string' ? l.payload.message : '', + ts, + textSources(l.payload, 'message'), + 'pre-task-prefix', + ) + return + } + // Only the turn shape is decoded here: a prefix `FileChange` item would + // parse a whole diff this walk never records. + if (l.type === 'event_msg' && l.payload?.type === 'item_completed' && l.payload.item?.type === 'UserMessage') { + const completed = codexCompletedItem(l) + if (completed?.type === 'UserMessage') { + recordInheritedTurn( + completed.userMessage.text, + ts, + textSources(completed.item, 'content'), + 'pre-task-prefix', + ) + } + } + } const ensureSubagentSpan = ( threadId: string, agentPath: string, @@ -938,16 +1121,21 @@ export class CodexAdapter implements HarnessTraceAdapter { return existing } const subagentType = agentPath.split('/').filter(Boolean).at(-1) ?? 'subagent' + // The child thread's lifecycle, assembled from `sub_agent_activity` + // events — NOT a call the model issued. It was a TOOL span named + // `tool.Agent`, so every tool-call count (here and in any consumer that + // counts TOOL spans or `tool.name`) ran high by one per child thread. + // It is an AGENT span with no `tool.name`, marked synthesized, so a + // counter needs no name allowlist to get the model's tool calls right. const toolSpan = span({ traceId, spanId: `subagent:${threadId}`, parentSpanId: eventCallSpan?.span_id ?? lastLlm, - name: 'tool.Agent', - kind: 'TOOL', + name: 'subagent.lifecycle', + kind: 'AGENT', startTime: eventTime, service: SERVICE, agent: SERVICE, - tool: 'Agent', step, status: 'UNSET', extra: { @@ -958,6 +1146,9 @@ export class CodexAdapter implements HarnessTraceAdapter { agent_thread_id: threadId, }, }), + [SYNTHESIZED_SPAN_ATTR]: true, + [SYNTHESIZED_SOURCE_ATTR]: 'codex.sub_agent_activity', + 'traces.codex.subagent_type': subagentType, 'traces.codex.subagent_path': agentPath, 'traces.codex.subagent_thread_id': threadId, ...(!observedStart ? { 'traces.codex.subagent_start_missing': true } : {}), @@ -971,9 +1162,14 @@ export class CodexAdapter implements HarnessTraceAdapter { } let reachedCurrentTask = !selectedBoundary + let prefixTimestamp: string | undefined for await (const l of readJsonl(ref.path, jsonl)) { if (!reachedCurrentTask) { - if (!isCodexTaskBoundary(l, selectedBoundary!)) continue + if (!isCodexTaskBoundary(l, selectedBoundary!)) { + prefixTimestamp = validTimestamp(l.timestamp) ?? prefixTimestamp + recordInheritedContext(l, prefixTimestamp ?? root.start_time) + continue + } reachedCurrentTask = true if (options.taskScope === 'turn') { root.start_time = codexTaskBoundary(l)?.timestamp ?? root.start_time @@ -1006,6 +1202,13 @@ export class CodexAdapter implements HarnessTraceAdapter { awaitingModel.length = 0 } else if (l.type === 'event_msg' && l.payload?.type === 'token_count') { const u = l.payload.info?.last_token_usage + // Read before the delta gate: a `token_count` event that reports no + // per-turn delta still advances the harness's cumulative counter, and + // the last snapshot in scope is the session total. + const reportedTotal = l.payload.info?.total_token_usage + if (reportedTotal && reportedCount(reportedTotal.total_tokens) !== undefined) { + sessionTotalUsage = reportedTotal + } if (u && (u.input_tokens || u.output_tokens)) { const cumulative = l.payload.info?.total_token_usage const cumulativeSignature = cumulative ? tokenUsageSignature(cumulative) : undefined @@ -1033,6 +1236,10 @@ export class CodexAdapter implements HarnessTraceAdapter { lastLlm = id step += 1 } + } else if (l.type === 'compacted') { + // Compaction replaces the model's context with a summary and a retained + // history. Both are inherited context, not new turns in this scope. + recordInheritedContext(l, ts) } else if ( l.type === 'response_item' && (l.payload?.type === 'function_call' || l.payload?.type === 'custom_tool_call') @@ -1293,8 +1500,33 @@ export class CodexAdapter implements HarnessTraceAdapter { Object.fromEntries([...skippedItemCounts].sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))), ) } + if (sessionTotalUsage) { + const total = reportedCount(sessionTotalUsage.total_tokens) + if (total !== undefined) { + root.attributes[SESSION_TOTAL_TOKENS] = total + root.attributes[SESSION_TOTAL_TOKENS_SOURCE] = CODEX_TOTAL_TOKENS_SOURCE + // The rest of the same snapshot, so the parts and the total agree. + const input = reportedCount(sessionTotalUsage.input_tokens) + const output = reportedCount(sessionTotalUsage.output_tokens) + const reasoning = reportedCount(sessionTotalUsage.reasoning_output_tokens) + const cached = reportedCount(sessionTotalUsage.cached_input_tokens) + if (input !== undefined) root.attributes[SESSION_TOTAL_INPUT_TOKENS] = input + if (output !== undefined) root.attributes[SESSION_TOTAL_OUTPUT_TOKENS] = output + if (reasoning !== undefined) root.attributes[SESSION_TOTAL_REASONING_TOKENS] = reasoning + if (cached !== undefined) root.attributes[SESSION_TOTAL_CACHED_INPUT_TOKENS] = cached + } + } + if (inheritedSpansEmitted > 0) root.attributes[INHERITED_SPAN_COUNT_ATTR] = inheritedSpansEmitted + // A dropped record stays visible as a count: inherited context is bounded, + // and "the cap was hit" must not read as "there was nothing before this". + if (inheritedSpansOmitted > 0) root.attributes[INHERITED_SPANS_OMITTED_ATTR] = inheritedSpansOmitted if (selectedBoundary?.turnId) { - for (const item of spans) item.attributes['traces.codex.turn_id'] ??= selectedBoundary.turnId + // An inherited record predates the selected turn; stamping it with that + // turn id would claim it happened inside the turn. + for (const item of spans) { + if (isInheritedSpan(item.attributes)) continue + item.attributes['traces.codex.turn_id'] ??= selectedBoundary.turnId + } } closeSpanAt(root, lastTimestamp ?? root.start_time) normalizeCodexIds(spans) diff --git a/src/adapters/provenance.ts b/src/adapters/provenance.ts new file mode 100644 index 0000000..d9a3405 --- /dev/null +++ b/src/adapters/provenance.ts @@ -0,0 +1,47 @@ +/** + * Provenance markers for spans an adapter did NOT take from an action the agent + * performed inside the parsed scope. + * + * Two cases, and both used to be invisible: + * + * - SYNTHESIZED — the adapter built the span from harness lifecycle events, + * not from a call the model issued. A Codex subagent's start/finish stream + * is one span per child thread; recording it as a TOOL call named + * `tool.Agent` made every tool-call count high by exactly the number of + * child threads, and nothing on the span let a counter tell the difference. + * - INHERITED — the record belongs to context this session carries but did + * not produce: the prefix a fork copies from its parent, and the history a + * `compacted` record retains. Keeping the human's words is the point; + * counting them as turns of THIS scope is not. + * + * Both markers are additive attributes. A count that means "what the agent did + * in this scope" filters them out; a reader that wants the context selects them. + */ + +/** `true` on a span the adapter synthesized from lifecycle events. */ +export const SYNTHESIZED_SPAN_ATTR = 'traces.span.synthesized' + +/** What the synthesized span was built from, e.g. `codex.sub_agent_activity`. */ +export const SYNTHESIZED_SOURCE_ATTR = 'traces.span.synthesized_from' + +/** `true` on a span carrying context from outside the parsed scope. */ +export const INHERITED_SPAN_ATTR = 'traces.session.inherited' + +/** Where the inherited record came from: `pre-task-prefix` or `compacted`. */ +export const INHERITED_SOURCE_ATTR = 'traces.session.inherited_source' + +export type InheritedSpanSource = 'pre-task-prefix' | 'compacted' + +/** Count of inherited spans in the batch, stamped on the root span. */ +export const INHERITED_SPAN_COUNT_ATTR = 'traces.session.inherited_span_count' + +/** Inherited records the adapter's per-session cap dropped, stamped on the root. */ +export const INHERITED_SPANS_OMITTED_ATTR = 'traces.session.inherited_spans_omitted' + +export function isSynthesizedSpan(attributes: Readonly>): boolean { + return attributes[SYNTHESIZED_SPAN_ATTR] === true +} + +export function isInheritedSpan(attributes: Readonly>): boolean { + return attributes[INHERITED_SPAN_ATTR] === true +} diff --git a/src/adoption.ts b/src/adoption.ts index abe5758..1f142db 100644 --- a/src/adoption.ts +++ b/src/adoption.ts @@ -13,6 +13,7 @@ import { readFile } from 'node:fs/promises' import { join } from 'node:path' +import { isSynthesizedSpan } from './adapters/provenance.js' import { toolArgumentsFromAttributes } from './adapters/tool-io.js' import { indexSessionIdsByTrace, @@ -299,12 +300,17 @@ export async function analyzeAdoption(spans: readonly OtlpSpan[], opts: Adoption addCounts(skillDocumentReads, skillDocuments) } const tn = toolName(s) + // A Codex subagent's lifecycle span is synthesized from harness events, so + // it carries no `tool.name`. It still reports one child thread, which is + // what the canonical subagent count means. + const subagentLifecycle = isSynthesizedSpan(s.attributes) + && typeof s.attributes['traces.codex.subagent_thread_id'] === 'string' if (tn === 'Skill') { sessionCapabilities.set(group, 'supported') const name = skillNameOf(parseInput(s)) skillInvocations[name] = (skillInvocations[name] ?? 0) + 1 sessionsWithSkill.add(group) - } else if (tn === 'Task' || tn === 'Agent') { + } else if (tn === 'Task' || tn === 'Agent' || subagentLifecycle) { const type = subagentTypeOf(parseInput(s)) subagentSpawns[type] = (subagentSpawns[type] ?? 0) + 1 sessionsWithSubagent.add(group) diff --git a/src/evidence.ts b/src/evidence.ts index 9871226..bffb648 100644 --- a/src/evidence.ts +++ b/src/evidence.ts @@ -6,6 +6,7 @@ import { OPENINFERENCE_SPAN_KIND, TOOL_NAME, } from '@tangle-network/agent-eval/trace-attributes' +import { isInheritedSpan, isSynthesizedSpan } from './adapters/provenance.js' import { ATTR } from './attributes.js' import { summarizeSpanExecution } from './execution.js' import type { OtlpSpan } from './otlp.js' @@ -113,6 +114,11 @@ function spanKind(span: OtlpSpan): string | undefined { return stringAttr(span, OPENINFERENCE_SPAN_KIND) } +/** A tool call the model issued — the only thing "tool call count" may mean. */ +function isModelToolCall(span: OtlpSpan): boolean { + return spanKind(span) === 'TOOL' && !isSynthesizedSpan(span.attributes) +} + function repoFromSpans(spans: readonly OtlpSpan[]): PolicyEvidenceRecord['repo'] { const attrs: { subjectKey?: string @@ -134,8 +140,15 @@ function repoFromSpans(spans: readonly OtlpSpan[]): PolicyEvidenceRecord['repo'] return attrs } +/** + * The window this session ACTED in. An inherited span carries context from + * before the parsed scope (a fork prefix, a compacted history), so counting its + * timestamps here would stretch the session window over the parent's work — + * and `buildSessionBundle` joins external evidence by exactly this window. + */ function timeBounds(spans: readonly OtlpSpan[]): { firstSpanAt: string | null; lastSpanAt: string | null } { const times = spans + .filter((span) => !isInheritedSpan(span.attributes)) .flatMap((span) => [span.start_time, span.end_time]) .filter((value) => value && value !== 'now') .sort() @@ -145,10 +158,15 @@ function timeBounds(spans: readonly OtlpSpan[]): { firstSpanAt: string | null; l } } +/** + * One row per tool the MODEL called. A synthesized span (a subagent's lifecycle + * assembled from harness events) is not a call the model issued, so it is + * excluded here and from every count below. + */ function summarizeTools(spans: readonly OtlpSpan[]): PolicyEvidenceToolSummary[] { const byTool = new Map() for (const span of spans) { - if (spanKind(span) !== 'TOOL') continue + if (!isModelToolCall(span)) continue const name = stringAttr(span, TOOL_NAME) ?? span.name.replace(/^tool\./, '') const current = byTool.get(name) ?? { calls: 0, errors: 0 } current.calls += 1 @@ -168,7 +186,7 @@ export async function buildPolicyEvidenceRecord( if (opts.sourceSha256 && !/^[a-f0-9]{64}$/.test(opts.sourceSha256)) { throw new Error('sourceSha256 must be a lowercase SHA-256 hex digest') } - const toolSpans = spans.filter((span) => spanKind(span) === 'TOOL') + const toolSpans = spans.filter(isModelToolCall) const erroredToolCallCount = toolSpans.filter((span) => span.status.code === 'ERROR').length const pipelines = await runPipelines(spans, { minLoopOccurrences: opts.minLoopOccurrences }) const loopLimit = opts.maxLoopExamples ?? 25 diff --git a/src/index.ts b/src/index.ts index 58bc6a0..73640bb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -79,6 +79,19 @@ export { textIsSynthetic, } from './adapters/actor.js' export type { Reaction } from './adapters/actor.js' +// Span provenance: which spans an adapter synthesized, and which carry context +// from outside the parsed scope. A count of what the agent DID excludes both. +export { + INHERITED_SOURCE_ATTR, + INHERITED_SPAN_ATTR, + INHERITED_SPAN_COUNT_ATTR, + INHERITED_SPANS_OMITTED_ATTR, + isInheritedSpan, + isSynthesizedSpan, + SYNTHESIZED_SOURCE_ATTR, + SYNTHESIZED_SPAN_ATTR, +} from './adapters/provenance.js' +export type { InheritedSpanSource } from './adapters/provenance.js' // ── Detection / analysis (built-in, or bring your own analysts) ─────────── export * from './failure-followup.js' // classifyFailureFollowUps() — blind vs adapted retry split diff --git a/src/live.ts b/src/live.ts index 1cac7f3..0679d19 100644 --- a/src/live.ts +++ b/src/live.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto' +import { isInheritedSpan, isSynthesizedSpan } from './adapters/provenance.js' import { isInnerToolCall } from './adapters/tool-io.js' import type { OtlpSpan } from './otlp.js' import type { PipelineReport } from './pipelines.js' @@ -238,12 +239,20 @@ function toolSignature(span: OtlpSpan): string { return `${toolName(span)}:${content || span.name}` } +/** + * A tool call the model issued. A synthesized span (a subagent lifecycle the + * adapter assembled from harness events) matches every surface heuristic below, + * so it is rejected first — otherwise this batch's tool count, and the error + * ratio derived from it, run high by one per child thread. + */ function isTool(span: OtlpSpan): boolean { + if (isSynthesizedSpan(span.attributes)) return false return spanKind(span) === 'TOOL' || span.attributes['tool.name'] != null || span.name.startsWith('tool.') } /** Command and file-change records inside a tool call carry tool I/O, not prose. */ function isTextSpan(span: OtlpSpan): boolean { + if (isSynthesizedSpan(span.attributes) || isInheritedSpan(span.attributes)) return false return !isTool(span) && !isInnerToolCall(span.attributes) && spanContent(span).length > 0 } diff --git a/src/reactions.ts b/src/reactions.ts index 4a5ce1e..b6df0ac 100644 --- a/src/reactions.ts +++ b/src/reactions.ts @@ -18,6 +18,7 @@ import { classifyReaction, CORRECTIVE_REACTIONS, type Reaction } from './adapters/actor.js' import { ACTOR_ATTR } from './adapters/conversation.js' +import { isInheritedSpan } from './adapters/provenance.js' import type { OtlpSpan } from './otlp.js' /** Reaction labels in stable render order. */ @@ -80,7 +81,10 @@ function isAssistant(s: OtlpSpan): boolean { return kind === 'LLM' || s.name.startsWith('message.assistant') } +/** An inherited turn was typed into ANOTHER scope (a fork's parent, a + * compacted history), so it has no assistant turn here to react to. */ function isHumanPrompt(s: OtlpSpan): boolean { + if (isInheritedSpan(s.attributes)) return false return s.name === 'user.prompt' && s.attributes[ACTOR_ATTR] === 'human' } diff --git a/src/report.ts b/src/report.ts index 1f35216..a1c9377 100644 --- a/src/report.ts +++ b/src/report.ts @@ -14,6 +14,7 @@ import type { } from '@tangle-network/agent-eval/contract' import type { AdoptionReport } from './adoption.js' import { ACTOR_ATTR } from './adapters/conversation.js' +import { isInheritedSpan } from './adapters/provenance.js' import { ATTR, sessionIdFromAttributes } from './attributes.js' import { incompleteInputsNote, type UnavailableCapabilities } from './conformance.js' import type { LoopConvergenceReport, SteeringChainReport } from './loop-analysis.js' @@ -91,9 +92,12 @@ export function sessionReportSource( sessionIdOverride?: string, ): ReportSource { const root = spans.find((item) => item.parent_span_id === null) ?? spans[0] - const prompt = spans.find( + // The subject names what THIS scope was asked to do, so an inherited turn + // (a fork's parent prompt, a compacted history) never supplies it. + const inScope = spans.filter((item) => !isInheritedSpan(item.attributes)) + const prompt = inScope.find( (item) => item.name === 'user.prompt' && item.attributes[ACTOR_ATTR] === 'human', - ) ?? spans.find((item) => item.name === 'user.prompt') ?? spans.find( + ) ?? inScope.find((item) => item.name === 'user.prompt') ?? inScope.find( (item) => item.attributes['span.type'] === 'interaction' && typeof item.attributes.content === 'string', ) const content = typeof prompt?.attributes.content === 'string' ? prompt.attributes.content : '' diff --git a/src/run-span-tree.ts b/src/run-span-tree.ts index f5a8d45..85659ea 100644 --- a/src/run-span-tree.ts +++ b/src/run-span-tree.ts @@ -39,6 +39,7 @@ import { LLM_OUTPUT_TOKEN_ATTR_KEYS, SPAN_KIND_ATTR_KEYS, } from '@tangle-network/agent-eval/trace-attributes' +import { isSynthesizedSpan } from './adapters/provenance.js' import { exportTraceEvidenceFile } from './file-export.js' import type { OtlpSpan } from './otlp.js' import { connectors, forEachTreeNode, int, ms, tokens, usd } from './run-view-format.js' @@ -291,7 +292,10 @@ export function buildSpanRunTree(spans: readonly OtlpSpan[], source: string): Sp llmSpansWithoutTokens += 1 } if (firstNumberAttr(span.attributes, LLM_COST_ATTR_KEYS) === null) llmSpansWithoutCost += 1 - } else if (kind === 'TOOL') host.toolCalls += 1 + } else if (kind === 'TOOL' && !isSynthesizedSpan(span.attributes)) { + // A synthesized lifecycle span is not a call this node made. + host.toolCalls += 1 + } host.startMs = Math.min(host.startMs, epoch(span.start_time)) host.endMs = Math.max(host.endMs, epoch(span.end_time)) if (span.status.code === 'ERROR') { diff --git a/tests/adapters.test.ts b/tests/adapters.test.ts index 3634b6b..e2e4bdf 100644 --- a/tests/adapters.test.ts +++ b/tests/adapters.test.ts @@ -1800,8 +1800,16 @@ describe('codex current tool and subagent events', () => { 'traces.codex.task_scope': 'fork-current', 'traces.codex.turn_id': currentTurnId, }) - expect(spans.filter((item) => item.name === 'user.prompt').map((item) => item.attributes.content)) + const prompts = spans.filter((item) => item.name === 'user.prompt') + expect(prompts.filter((item) => item.attributes['traces.session.inherited'] !== true) + .map((item) => item.attributes.content)) .toEqual(['current child prompt']) + // The pre-fork prefix is kept, marked, and left out of this turn's identity. + const inherited = prompts.filter((item) => item.attributes['traces.session.inherited'] === true) + expect(inherited.map((item) => item.attributes.content)).toEqual(['inherited parent prompt']) + expect(inherited[0]?.attributes['traces.session.inherited_source']).toBe('pre-task-prefix') + expect(inherited[0]?.attributes['traces.codex.turn_id']).toBeUndefined() + expect(spans[0]?.attributes['traces.session.inherited_span_count']).toBe(1) }) it('uses task timestamps when older child events omit started_at', async () => { @@ -2365,7 +2373,9 @@ describe('codex current tool and subagent events', () => { const spans = await new CodexAdapter().parse(refFor(path, 'codex')) const tools = spans.filter((item) => item.attributes['openinference.span.kind'] === 'TOOL') - expect(tools).toHaveLength(10) + // The two subagent lifecycles are AGENT spans, not calls the model made. + expect(tools).toHaveLength(8) + expect(tools.every((item) => item.attributes['traces.span.synthesized'] === undefined)).toBe(true) const verifications = tools.filter((item) => item.attributes['tool.name'] === 'exec_command.verify') expect(verifications).toHaveLength(2) const failedVerification = verifications.find((item) => item.status.code === 'ERROR') @@ -2401,8 +2411,12 @@ describe('codex current tool and subagent events', () => { expect(writeStdin?.attributes['traces.expected_blocking']).toBe(true) expect(writeStdin?.status.code).toBe('OK') - const agents = tools.filter((item) => item.attributes['tool.name'] === 'Agent') + const agents = spans.filter((item) => item.attributes['traces.span.synthesized'] === true) expect(agents).toHaveLength(2) + expect(agents.every((item) => item.name === 'subagent.lifecycle')).toBe(true) + expect(agents.every((item) => item.attributes['openinference.span.kind'] === 'AGENT')).toBe(true) + expect(agents.every((item) => item.attributes['tool.name'] === undefined)).toBe(true) + expect(agents.every((item) => item.attributes['traces.span.synthesized_from'] === 'codex.sub_agent_activity')).toBe(true) const agent = agents.find((item) => String(item.attributes['input.value']).includes('paper_audit')) expect(JSON.parse(String(agent?.attributes['input.value']))).toEqual({ subagent_type: 'paper_audit', diff --git a/tests/codex-token-and-provenance.test.ts b/tests/codex-token-and-provenance.test.ts new file mode 100644 index 0000000..e1d5081 --- /dev/null +++ b/tests/codex-token-and-provenance.test.ts @@ -0,0 +1,365 @@ +/** + * Synthetic Codex rollouts for three adapter facts an audit question asks about: + * the harness's own cumulative token total, which spans are calls the model + * made, and the human context a forked or compacted session inherited. + * + * Every rollout here is written inline. None comes from a recorded session. + */ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, describe, expect, it } from 'vitest' +import { CodexAdapter } from '../src/adapters/codex.js' +import { + INHERITED_SPAN_ATTR, + INHERITED_SPAN_COUNT_ATTR, + INHERITED_SPANS_OMITTED_ATTR, + isSynthesizedSpan, + SYNTHESIZED_SPAN_ATTR, +} from '../src/adapters/provenance.js' +import { buildPolicyEvidenceRecord } from '../src/evidence.js' +import { analyzeLiveBatch } from '../src/live.js' +import type { OtlpSpan } from '../src/otlp.js' +import { runPipelines } from '../src/pipelines.js' +import { sessionReportSource } from '../src/report.js' +import { at, ms, type Row, writeRollout } from './codex-facts-fixture.js' + +const dir = mkdtempSync(join(tmpdir(), 'traces-codex-provenance-')) +afterAll(() => rmSync(dir, { recursive: true, force: true })) + +const toolCall = (t: number, callId: string, name: string, cmd: string): Row => ({ + t, + type: 'response_item', + payload: { type: 'function_call', call_id: callId, name, arguments: JSON.stringify({ cmd }) }, +}) +const toolOutput = (t: number, callId: string): Row => ({ + t, + type: 'response_item', + payload: { type: 'function_call_output', call_id: callId, output: { exit_code: 0, output: 'ok' } }, +}) +const subagentActivity = (t: number, threadId: string, agentPath: string, kind: string): Row => ({ + t, + type: 'event_msg', + payload: { + type: 'sub_agent_activity', + event_id: `${threadId}-${kind}`, + occurred_at_ms: ms(t), + agent_thread_id: threadId, + agent_path: agentPath, + kind, + }, +}) +const tokenCount = ( + t: number, + last: Record | undefined, + total: Record, +): Row => ({ + t, + type: 'event_msg', + payload: { + type: 'token_count', + info: { ...(last ? { last_token_usage: last } : {}), total_token_usage: total }, + }, +}) +const userMessage = (t: number, text: string): Row => ({ + t, + type: 'response_item', + payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text }] }, +}) +const compacted = (t: number, summary: string, window: number, history: readonly string[]): Row => ({ + t, + type: 'compacted', + payload: { + message: summary, + window_number: window, + window_id: `window-${window}`, + previous_window_id: `window-${window - 1}`, + first_window_id: 'window-0', + replacement_history: [ + { type: 'message', id: `dev-${window}`, role: 'developer', content: [{ type: 'input_text', text: 'developer scaffolding' }] }, + ...history.map((text, index) => ({ + type: 'message', + id: `hist-${window}-${index}`, + role: 'user', + content: [{ type: 'input_text', text }], + })), + ], + }, +}) + +const parse = (ref: Parameters[0]): Promise => + new CodexAdapter().parse(ref) + +const kindOf = (span: OtlpSpan): unknown => span.attributes['openinference.span.kind'] +const inherited = (spans: readonly OtlpSpan[]): OtlpSpan[] => + spans.filter((span) => span.attributes[INHERITED_SPAN_ATTR] === true) + +describe('Codex cumulative token total', () => { + it('carries the harness counter verbatim instead of deriving one', async () => { + const ref = writeRollout(dir, 'token-counter', [ + { t: 0, type: 'session_meta', payload: { id: 'token-session', cwd: '/workspace/demo' } }, + { t: 0.5, type: 'turn_context', payload: { cwd: '/workspace/demo', model: 'gpt-fixture' } }, + { t: 1, type: 'event_msg', payload: { type: 'task_started', turn_id: 'turn-1' } }, + tokenCount(2, { input_tokens: 1_000, output_tokens: 20 }, { input_tokens: 1_000, output_tokens: 20, total_tokens: 1_020 }), + // A repeat of the same cumulative snapshot: one turn, reported twice. + tokenCount(3, { input_tokens: 1_000, output_tokens: 20 }, { input_tokens: 1_000, output_tokens: 20, total_tokens: 1_020 }), + tokenCount( + 4, + { input_tokens: 2_000, output_tokens: 30 }, + { input_tokens: 3_000, output_tokens: 50, reasoning_output_tokens: 10, cached_input_tokens: 500, total_tokens: 3_050 }, + ), + // The counter's last word: the harness advanced the total with no + // per-turn delta to report, so no `llm.turn` span carries this number. + tokenCount( + 5, + undefined, + { input_tokens: 4_000, output_tokens: 60, reasoning_output_tokens: 12, cached_input_tokens: 700, total_tokens: 4_100 }, + ), + { t: 6, type: 'event_msg', payload: { type: 'task_complete', turn_id: 'turn-1' } }, + ]) + const spans = await parse(ref) + const root = spans[0]! + + expect(root.attributes['traces.session.total_tokens']).toBe(4_100) + expect(root.attributes['traces.session.total_tokens_source']).toBe('codex.token_count.info.total_token_usage') + expect(root.attributes['traces.session.total_input_tokens']).toBe(4_000) + expect(root.attributes['traces.session.total_output_tokens']).toBe(60) + expect(root.attributes['traces.session.total_reasoning_tokens']).toBe(12) + expect(root.attributes['traces.session.total_cached_input_tokens']).toBe(700) + + // Neither number a reader could compute from the spans equals the total: + // the deltas sum to 3,050 and the snapshots sum to 9,270. + const turns = spans.filter((span) => span.name === 'llm.turn') + expect(turns).toHaveLength(2) + const deltaSum = turns.reduce( + (total, span) => + total + + Number(span.attributes['llm.token_count.prompt'] ?? 0) + + Number(span.attributes['llm.token_count.completion'] ?? 0), + 0, + ) + expect(deltaSum).toBe(3_050) + expect(root.attributes['traces.session.total_tokens']).not.toBe(deltaSum) + }) + + it('records no total when the harness reported none', async () => { + const ref = writeRollout(dir, 'token-counter-absent', [ + { t: 0, type: 'session_meta', payload: { id: 'token-absent', cwd: '/workspace/demo' } }, + { t: 1, type: 'event_msg', payload: { type: 'task_started', turn_id: 'turn-1' } }, + { t: 2, type: 'event_msg', payload: { type: 'token_count', info: { last_token_usage: { input_tokens: 10, output_tokens: 2 } } } }, + { t: 3, type: 'event_msg', payload: { type: 'task_complete', turn_id: 'turn-1' } }, + ]) + const spans = await parse(ref) + // Missing stays missing: an unknown total must not become zero. + expect(spans[0]!.attributes['traces.session.total_tokens']).toBeUndefined() + expect(spans[0]!.attributes['traces.session.total_tokens_source']).toBeUndefined() + }) +}) + +describe('Codex synthesized subagent spans', () => { + const subagentRollout = () => + writeRollout(dir, 'synthesized-subagents', [ + { t: 0, type: 'session_meta', payload: { id: 'synth-session', cwd: '/workspace/demo' } }, + { t: 0.5, type: 'turn_context', payload: { cwd: '/workspace/demo', model: 'gpt-fixture' } }, + { t: 1, type: 'event_msg', payload: { type: 'task_started', turn_id: 'turn-1' } }, + userMessage(1.5, 'Audit the parser and report back.'), + tokenCount(2, { input_tokens: 100, output_tokens: 10 }, { input_tokens: 100, output_tokens: 10, total_tokens: 110 }), + toolCall(3, 'call-1', 'exec_command', 'rm -rf build'), + toolOutput(3.5, 'call-1'), + toolCall(4, 'call-2', 'spawn_agent', 'parser_audit'), + toolOutput(4.5, 'call-2'), + toolCall(5, 'call-3', 'exec_command', 'curl -X POST https://example.test/hook'), + toolOutput(5.5, 'call-3'), + subagentActivity(6, 'thread-a', '/root/parser_audit', 'started'), + subagentActivity(7, 'thread-b', '/root/runtime_audit', 'started'), + subagentActivity(8, 'thread-a', '/root/parser_audit', 'completed'), + subagentActivity(9, 'thread-b', '/root/runtime_audit', 'completed'), + { t: 10, type: 'event_msg', payload: { type: 'task_complete', turn_id: 'turn-1' } }, + ]) + + it('keeps the lifecycle span out of every tool-call count', async () => { + const spans = await parse(subagentRollout()) + const synthesized = spans.filter((span) => isSynthesizedSpan(span.attributes)) + const toolSpans = spans.filter((span) => kindOf(span) === 'TOOL') + + expect(synthesized).toHaveLength(2) + expect(synthesized.map((span) => span.name)).toEqual(['subagent.lifecycle', 'subagent.lifecycle']) + expect(synthesized.every((span) => kindOf(span) === 'AGENT')).toBe(true) + expect(synthesized.every((span) => span.attributes['tool.name'] === undefined)).toBe(true) + expect(synthesized.map((span) => span.attributes['traces.codex.subagent_type'])) + .toEqual(['parser_audit', 'runtime_audit']) + + // Before and after, on the same rollout. The previous schema counted a span + // as a tool call when it was TOOL-kind, which included both lifecycle spans. + const countedBefore = spans.filter( + (span) => kindOf(span) === 'TOOL' || isSynthesizedSpan(span.attributes), + ).length + const countedAfter = toolSpans.length + expect(countedBefore - countedAfter).toBe(synthesized.length) + expect(countedAfter).toBe(3) + expect(toolSpans.map((span) => span.attributes['tool.name'])) + .toEqual(['exec_command', 'spawn_agent', 'exec_command']) + }) + + it('reports the model-issued count through the evidence, live, and pipeline paths', async () => { + const ref = subagentRollout() + const spans = await parse(ref) + + const record = await buildPolicyEvidenceRecord(ref, spans) + expect(record.metrics.toolCallCount).toBe(3) + expect(record.metrics.tools.map((tool) => tool.name).sort()).toEqual(['exec_command', 'spawn_agent']) + expect(record.metrics.tools.find((tool) => tool.name === 'Agent')).toBeUndefined() + + expect(analyzeLiveBatch(spans).toolCallCount).toBe(3) + + const pipelines = await runPipelines(spans) + expect(pipelines.toolUse.reduce((total, run) => total + run.totalCalls, 0)).toBe(3) + }) +}) + +describe('Codex inherited context', () => { + const CHILD_ID = 'child-thread-1' + const PARENT_ASK = 'Start with the parser, not the reporter.' + const PARENT_FOLLOW_UP = 'Keep the fixture list short.' + + const forkRollout = () => + writeRollout(dir, 'fork-inherited', [ + { + t: 0, + type: 'session_meta', + payload: { + id: CHILD_ID, + cwd: '/workspace/demo', + thread_source: 'subagent', + source: { subagent: { thread_spawn: { parent_thread_id: 'parent-thread-1', depth: 1, agent_path: '/root/parser_audit' } } }, + }, + }, + { t: 0.5, type: 'turn_context', payload: { cwd: '/workspace/demo', model: 'gpt-fixture' } }, + // The parent's task, copied into the fork's prefix. + { t: 1, type: 'event_msg', payload: { type: 'task_started', turn_id: 'parent-turn-1' } }, + userMessage(2, PARENT_ASK), + { t: 3, type: 'event_msg', payload: { type: 'user_message', message: PARENT_FOLLOW_UP } }, + compacted(4, 'Summary of the first window.', 1, [PARENT_ASK, PARENT_FOLLOW_UP]), + // A second compaction repeats the same retained turns. + compacted(5, 'Summary of the second window.', 2, [PARENT_ASK, 'and keep the budget under an hour.']), + toolCall(6, 'parent-call-1', 'exec_command', 'rm -rf parent-build'), + toolOutput(6.5, 'parent-call-1'), + // The fork boundary: everything below is this session's own scope. + { t: 10, type: 'event_msg', payload: { type: 'task_started', turn_id: CHILD_ID } }, + userMessage(11, 'Audit the parser and report back.'), + tokenCount(12, { input_tokens: 100, output_tokens: 10 }, { input_tokens: 100, output_tokens: 10, total_tokens: 110 }), + toolCall(13, 'child-call-1', 'exec_command', 'rm -rf child-build'), + toolOutput(13.5, 'child-call-1'), + { t: 14, type: 'event_msg', payload: { type: 'task_complete', turn_id: CHILD_ID } }, + ]) + + it('keeps the pre-fork prefix and compacted history as marked spans', async () => { + const ref = forkRollout() + const spans = await new CodexAdapter().parse(ref, { captureSources: true }) + const root = spans[0]! + expect(root.attributes['traces.codex.task_scope']).toBe('fork-current') + + const inheritedSpans = inherited(spans) + expect(root.attributes[INHERITED_SPAN_COUNT_ATTR]).toBe(inheritedSpans.length) + + const inheritedPrompts = inheritedSpans.filter((span) => span.name === 'user.prompt') + // The human's words reach a span, once each, however many records repeat them. + expect(inheritedPrompts.map((span) => span.attributes.content)).toEqual([ + PARENT_ASK, + PARENT_FOLLOW_UP, + 'and keep the budget under an hour.', + ]) + expect(inheritedPrompts.every((span) => span.attributes['tangle.actor'] === 'human')).toBe(true) + expect(inheritedPrompts.map((span) => span.attributes['traces.session.inherited_source'])).toEqual([ + 'pre-task-prefix', + 'pre-task-prefix', + 'compacted', + ]) + // Every inherited quote cites the record it came from. + expect(inheritedPrompts.every((span) => typeof span.attributes['traces.source_record.content'] === 'string')).toBe(true) + + const compactions = inheritedSpans.filter((span) => span.name === 'session.compacted') + expect(compactions.map((span) => span.attributes.content)).toEqual([ + 'Summary of the first window.', + 'Summary of the second window.', + ]) + expect(compactions.map((span) => span.attributes['traces.codex.compaction_window_number'])).toEqual([1, 2]) + expect(compactions[0]!.attributes['traces.codex.compaction_window_id']).toBe('window-1') + }) + + it('leaves this scope own counts and identity untouched', async () => { + const ref = forkRollout() + const spans = await parse(ref) + + const ownPrompts = spans.filter( + (span) => span.name === 'user.prompt' && span.attributes[INHERITED_SPAN_ATTR] !== true, + ) + expect(ownPrompts.map((span) => span.attributes.content)).toEqual(['Audit the parser and report back.']) + // A forked child received its brief from its parent agent, not a person. + expect(ownPrompts[0]!.attributes['tangle.actor']).toBe('agent') + + // The prefix's tool call belongs to the parent's turn and is not parsed. + const toolSpans = spans.filter((span) => kindOf(span) === 'TOOL') + expect(toolSpans.map((span) => span.attributes['input.value'])).toEqual([ + JSON.stringify({ cmd: 'rm -rf child-build' }), + ]) + + const record = await buildPolicyEvidenceRecord(ref, spans) + expect(record.metrics.toolCallCount).toBe(1) + // The acted-in window starts at the fork, not at the parent's first record. + expect(record.metrics.firstSpanAt).toBe(at(10)) + + // The report subject names what THIS scope was asked to do. + expect(sessionReportSource(ref, spans).subject).toBe('Audit the parser and report back.') + }) + + it('counts the inherited records its per-session cap dropped', async () => { + const rows: Row[] = [ + { t: 0, type: 'session_meta', payload: { id: 'cap-session', cwd: '/workspace/demo' } }, + { t: 1, type: 'event_msg', payload: { type: 'task_started', turn_id: 'turn-1' } }, + ] + // 260 distinct inherited turns against a cap of 200. + for (let index = 0; index < 260; index += 1) { + rows.push(userMessage(2 + index * 0.001, `inherited turn ${index}`)) + } + rows.push({ t: 3, type: 'event_msg', payload: { type: 'task_complete', turn_id: 'turn-1' } }) + rows.push({ t: 4, type: 'event_msg', payload: { type: 'task_started', turn_id: 'turn-2' } }) + rows.push(userMessage(5, 'the turn in scope')) + rows.push({ t: 6, type: 'event_msg', payload: { type: 'task_complete', turn_id: 'turn-2' } }) + const ref = writeRollout(dir, 'inherited-cap', rows) + + const spans = await new CodexAdapter().parse(ref, { taskScope: 'latest' }) + const root = spans[0]! + expect(root.attributes[INHERITED_SPAN_COUNT_ATTR]).toBe(200) + // What the cap turned away is reported, not silently dropped. + expect(root.attributes[INHERITED_SPANS_OMITTED_ATTR]).toBe(60) + expect(inherited(spans)).toHaveLength(200) + }) + + it('keeps a compacted record inside the parsed scope as inherited context', async () => { + const ref = writeRollout(dir, 'compaction-in-scope', [ + { t: 0, type: 'session_meta', payload: { id: 'compaction-session', cwd: '/workspace/demo' } }, + { t: 1, type: 'event_msg', payload: { type: 'task_started', turn_id: 'turn-1' } }, + userMessage(2, 'Ship the parser fix.'), + compacted(3, 'Summary of the first window.', 1, ['Ship the parser fix.', 'and add a regression test.']), + toolCall(4, 'call-1', 'exec_command', 'rm -rf build'), + toolOutput(4.5, 'call-1'), + { t: 5, type: 'event_msg', payload: { type: 'task_complete', turn_id: 'turn-1' } }, + ]) + const spans = await parse(ref) + const inheritedSpans = inherited(spans) + + expect(inheritedSpans.map((span) => span.name)).toEqual([ + 'session.compacted', + 'user.prompt', + 'user.prompt', + ]) + expect(inheritedSpans.every((span) => span.attributes['traces.session.inherited_source'] === 'compacted')).toBe(true) + // The turn this scope actually received keeps its own span. + const ownPrompts = spans.filter( + (span) => span.name === 'user.prompt' && span.attributes[INHERITED_SPAN_ATTR] !== true, + ) + expect(ownPrompts.map((span) => span.attributes.content)).toEqual(['Ship the parser fix.']) + expect(spans.filter((span) => span.attributes[SYNTHESIZED_SPAN_ATTR] === true)).toHaveLength(0) + }) +})