diff --git a/README.md b/README.md index c2f93ed..ff5d32c 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,10 @@ Two of those analyses are why the loop shape exists at all: caused which subsequent round. A link whose target is missing is kept and marked, because a shorter chain than the producer recorded is the opposite of the truth. +Parent and verdict lookups use both trace and span IDs. +Steering edges retain `causeTraceId` and `effectTraceId`, including unresolved targets. +Generic chat transcripts report unknown execution status because message text does not establish completion. + Nothing is repaired on the way in. A span with an unreadable timestamp is reported and excluded, because a synthesized value would put invented work into a total. diff --git a/package.json b/package.json index 576d186..9c1f38a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/traces", - "version": "0.12.3", + "version": "0.13.0", "description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.", "type": "module", "license": "MIT", diff --git a/src/chat-trajectory.ts b/src/chat-trajectory.ts index 8ed9b89..876e440 100644 --- a/src/chat-trajectory.ts +++ b/src/chat-trajectory.ts @@ -67,6 +67,8 @@ export function chatTrajectoryToSpans( spanId: 'root', name: 'trajectory', kind: 'AGENT', + // A transcript does not record whether the task or execution completed. + status: 'UNSET', startTime: times[0]!, endTime: times.at(-1)!, service, @@ -106,6 +108,7 @@ export function chatTrajectoryToSpans( parentSpanId: 'root', name: messageName(kind, message), kind, + status: 'UNSET', startTime: times[index]!, service, agent: messageAgent(message.role), diff --git a/src/conformance.ts b/src/conformance.ts index ba1605f..b26e270 100644 --- a/src/conformance.ts +++ b/src/conformance.ts @@ -153,7 +153,7 @@ export interface SpanStructure { * correct one-root trace narrate itself as broken. */ export function summarizeSpanStructure(spans: readonly OtlpSpan[]): SpanStructure { - const ids = new Set(spans.map((span) => span.span_id)) + const ids = new Set(spans.map((span) => JSON.stringify([span.trace_id, span.span_id]))) const rootsByTrace = new Map() let rootless = 0 let orphans = 0 @@ -164,7 +164,7 @@ export function summarizeSpanStructure(spans: readonly OtlpSpan[]): SpanStructur if (span.parent_span_id === null) { rootless += 1 rootsByTrace.set(span.trace_id, (rootsByTrace.get(span.trace_id) ?? 0) + 1) - } else if (!ids.has(span.parent_span_id)) orphans += 1 + } else if (!ids.has(JSON.stringify([span.trace_id, span.parent_span_id]))) orphans += 1 } let extraRoots = 0 let tracesWithoutRoot = 0 diff --git a/src/loop-analysis.ts b/src/loop-analysis.ts index c66564a..af000fa 100644 --- a/src/loop-analysis.ts +++ b/src/loop-analysis.ts @@ -69,11 +69,13 @@ export interface LoopConvergenceReport { /** One causal edge: a graded span, and the work it caused. */ export interface SteeringEdge { /** The span that CAUSED the work — a verdict, an earlier attempt. */ + readonly causeTraceId: string readonly causeSpanId: string readonly causeName: string | null readonly causeOutcome: Outcome | null readonly causeScore: number | null /** The span the link was recorded on: the work that was caused. */ + readonly effectTraceId: string readonly effectSpanId: string readonly effectName: string readonly effectIteration: number | null @@ -90,6 +92,10 @@ export interface SteeringChainReport { readonly byKind: Readonly> } +function spanKey(traceId: string, spanId: string): string { + return JSON.stringify([traceId, spanId]) +} + function attributes(span: OtlpSpan): Record { return span.attributes } @@ -131,18 +137,18 @@ function verdictFor( const ownScore = scoreOf(span) if (own !== null || ownScore !== null) return { outcome: own, score: ownScore, spanId: span.span_id } - const queue: OtlpSpan[] = [...(childrenOf.get(span.span_id) ?? [])] - const seen = new Set([span.span_id]) + const queue: OtlpSpan[] = [...(childrenOf.get(spanKey(span.trace_id, span.span_id)) ?? [])] + const seen = new Set([spanKey(span.trace_id, span.span_id)]) while (queue.length > 0) { const current = queue.shift()! - if (seen.has(current.span_id)) continue - seen.add(current.span_id) + if (seen.has(spanKey(current.trace_id, current.span_id))) continue + seen.add(spanKey(current.trace_id, current.span_id)) const outcome = outcomeOf(current) const score = scoreOf(current) if (outcome !== null || score !== null) return { outcome, score, spanId: current.span_id } // Do not descend past a round: a nested loop grades itself. if (iterationOf(current) !== null) continue - queue.push(...(childrenOf.get(current.span_id) ?? [])) + queue.push(...(childrenOf.get(spanKey(current.trace_id, current.span_id)) ?? [])) } return { outcome: null, score: null, spanId: null } } @@ -151,9 +157,10 @@ function childIndex(spans: readonly OtlpSpan[]): Map { const children = new Map() for (const span of spans) { if (span.parent_span_id === null) continue - const bucket = children.get(span.parent_span_id) ?? [] + const key = spanKey(span.trace_id, span.parent_span_id) + const bucket = children.get(key) ?? [] bucket.push(span) - children.set(span.parent_span_id, bucket) + children.set(key, bucket) } return children } @@ -273,7 +280,10 @@ export function analyzeLoopConvergence(spans: readonly OtlpSpan[]): LoopConverge */ export function analyzeSteeringChain(spans: readonly OtlpSpan[]): SteeringChainReport { const byId = new Map() - for (const span of spans) if (!byId.has(span.span_id)) byId.set(span.span_id, span) + for (const span of spans) { + const key = spanKey(span.trace_id, span.span_id) + if (!byId.has(key)) byId.set(key, span) + } const edges: SteeringEdge[] = [] const byKind: Record = {} @@ -286,13 +296,15 @@ export function analyzeSteeringChain(spans: readonly OtlpSpan[]): SteeringChainR const kindValue = link.attributes?.[LINK_KIND_ATTR] const linkKind = typeof kindValue === 'string' && kindValue.length > 0 ? kindValue : 'unspecified' byKind[linkKind] = (byKind[linkKind] ?? 0) + 1 - const cause = byId.get(link.span_id) + const cause = byId.get(spanKey(link.trace_id, link.span_id)) if (cause === undefined) dangling += 1 edges.push({ + causeTraceId: link.trace_id, causeSpanId: link.span_id, causeName: cause?.name ?? null, causeOutcome: cause ? outcomeOf(cause) : null, causeScore: cause ? scoreOf(cause) : null, + effectTraceId: span.trace_id, effectSpanId: span.span_id, effectName: span.name, effectIteration: iterationOf(span), diff --git a/tests/chat-trajectory.test.ts b/tests/chat-trajectory.test.ts index 6302bfc..3d76068 100644 --- a/tests/chat-trajectory.test.ts +++ b/tests/chat-trajectory.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { summarizeSpanExecution } from '../src/execution.js' import { chatTrajectoryToSpans } from '../src/chat-trajectory.js' import { exportTraceEvidenceRows } from '../src/file-export.js' @@ -31,6 +32,17 @@ const trajectory = { } describe('chatTrajectoryToSpans', () => { + it('does not treat an assistant completion claim as observed execution success', () => { + const spans = chatTrajectoryToSpans([ + { role: 'user', content: 'Implement and verify the requested change.' }, + { role: 'assistant', content: 'Everything is done.' }, + ]) + expect(spans.every((item) => item.status.code === 'UNSET')).toBe(true) + const report = summarizeSpanExecution(spans) + expect(report.execution.terminalOutcomes.succeeded).toBe(0) + expect(report.execution.terminalOutcomes.unknown).toBe(1) + }) + it('preserves assistant action ordinals and captured model usage', () => { const spans = chatTrajectoryToSpans(trajectory) diff --git a/tests/loop-analysis.test.ts b/tests/loop-analysis.test.ts index 7a3eb13..ed8c2a9 100644 --- a/tests/loop-analysis.test.ts +++ b/tests/loop-analysis.test.ts @@ -47,6 +47,17 @@ function loop(scores: readonly (number | null)[], outcomes?: readonly string[]): } describe('analyzeLoopConvergence', () => { + it('keeps reused round and verdict IDs scoped to their trace', () => { + const otherTrace = deriveHexId('other-loop', 16) + const first = loop([0.2, 0.9]) + const second = loop([0.8, 0.1]).map((item) => ({ ...item, trace_id: otherTrace })) + const report = analyzeLoopConvergence([...first, ...second]) + expect(report.loops.find((item) => item.traceId === TRACE)?.trend).toBe('improved') + const other = report.loops.find((item) => item.traceId === otherTrace) + expect(other?.trend).toBe('regressed') + expect(other?.iterations.map((item) => item.score)).toEqual([0.8, 0.1]) + }) + it('reads the verdict from the round\'s evaluator child and calls a rising score improved', () => { const report = analyzeLoopConvergence(loop([0.2, 0.5, 0.9], ['fail', 'fail', 'pass'])) @@ -117,6 +128,21 @@ describe('analyzeLoopConvergence', () => { }) describe('analyzeSteeringChain', () => { + it('resolves cross-trace links by both IDs and retains missing targets', () => { + const otherTrace = deriveHexId('other-loop', 16) + const absentTrace = deriveHexId('absent-loop', 16) + const wrongCause = span({ id: 'verdict', name: 'wrong', attributes: { [ATTR.score]: 0.2 } }) + const cause = { ...span({ id: 'verdict', name: 'actual', attributes: { [ATTR.score]: 0.9 } }), trace_id: otherTrace } + const effect = span({ id: 'retry', links: [ + { trace_id: otherTrace, span_id: cause.span_id }, + { trace_id: absentTrace, span_id: cause.span_id }, + ] }) + const report = analyzeSteeringChain([wrongCause, cause, effect]) + expect(report.dangling).toBe(1) + expect(report.edges[0]).toMatchObject({ causeTraceId: otherTrace, effectTraceId: TRACE, causeName: 'actual', causeScore: 0.9, resolved: true }) + expect(report.edges[1]).toMatchObject({ causeTraceId: absentTrace, effectTraceId: TRACE, causeName: null, causeScore: null, resolved: false }) + }) + it('reads the edge backwards from the caused round to the verdict that caused it', () => { const spans = [ ...loop([0.2, 0.8], ['fail', 'pass']), @@ -135,10 +161,12 @@ describe('analyzeSteeringChain', () => { expect(report.dangling).toBe(0) expect(report.byKind).toEqual({ steered_by: 1 }) expect(report.edges[0]).toEqual({ + causeTraceId: TRACE, causeSpanId: deriveHexId('verdict-0', 8), causeName: 'verification', causeOutcome: 'fail', causeScore: 0.2, + effectTraceId: TRACE, effectSpanId: deriveHexId('round-1', 8), effectName: 'round 2', effectIteration: 2, diff --git a/tests/otlp-input.test.ts b/tests/otlp-input.test.ts index 4857c5d..80e2dcc 100644 --- a/tests/otlp-input.test.ts +++ b/tests/otlp-input.test.ts @@ -477,6 +477,13 @@ describe('detection of contract-shaped OTLP', () => { }) describe('summarizeSpanStructure', () => { + it('does not resolve a missing parent from another trace', () => { + const source = contractTrace() + const parent = otlpRowToSpan(source[0], TRACE).span! + const child = { ...parent, trace_id: 'other-trace', span_id: 'child', parent_span_id: parent.span_id } + expect(summarizeSpanStructure([parent, child]).orphans).toBe(1) + }) + it('measures the degree of flatness the per-trace finding cannot express', async () => { const rows = [ ...contractTrace(),