From d0cb9d0d33c59e5abe19fb542dbbfe00ab01c334 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 16 Apr 2026 16:41:53 +0900 Subject: [PATCH 01/18] feat(core): Send gen_ai spans as v2 envelope items --- packages/core/src/client.ts | 9 + .../src/tracing/spans/extractGenAiSpans.ts | 46 ++++ .../tracing/spans/spanJsonToStreamedSpan.ts | 78 ++++++ packages/core/src/types-hoist/envelope.ts | 2 +- .../tracing/spans/extractGenAiSpans.test.ts | 156 ++++++++++++ .../spans/spanJsonToStreamedSpan.test.ts | 240 ++++++++++++++++++ 6 files changed, 530 insertions(+), 1 deletion(-) create mode 100644 packages/core/src/tracing/spans/extractGenAiSpans.ts create mode 100644 packages/core/src/tracing/spans/spanJsonToStreamedSpan.ts create mode 100644 packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts create mode 100644 packages/core/test/lib/tracing/spans/spanJsonToStreamedSpan.test.ts diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 6c3ca949f38e..b3b81d21ed47 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -12,6 +12,7 @@ import type { Scope } from './scope'; import { updateSession } from './session'; import { getDynamicSamplingContextFromScope } from './tracing/dynamicSamplingContext'; import { isStreamedBeforeSendSpanCallback } from './tracing/spans/beforeSendSpan'; +import { extractGenAiSpansFromEvent } from './tracing/spans/extractGenAiSpans'; import { DEFAULT_TRANSPORT_BUFFER_SIZE } from './transports/base'; import type { Breadcrumb, BreadcrumbHint, FetchBreadcrumbHint, XhrBreadcrumbHint } from './types-hoist/breadcrumb'; import type { CheckIn, MonitorConfig } from './types-hoist/checkin'; @@ -522,12 +523,20 @@ export abstract class Client { public sendEvent(event: Event, hint: EventHint = {}): void { this.emit('beforeSendEvent', event, hint); + // Extract gen_ai spans from transaction and convert to span v2 format. + // This mutates event.spans to remove the extracted spans. + const genAiSpanItem = extractGenAiSpansFromEvent(event, this); + let env = createEventEnvelope(event, this._dsn, this._options._metadata, this._options.tunnel); for (const attachment of hint.attachments || []) { env = addItemToEnvelope(env, createAttachmentEnvelopeItem(attachment)); } + if (genAiSpanItem) { + env = addItemToEnvelope(env, genAiSpanItem); + } + // sendEnvelope should not throw // eslint-disable-next-line @typescript-eslint/no-floating-promises this.sendEnvelope(env).then(sendResponse => this.emit('afterSendEvent', event, sendResponse)); diff --git a/packages/core/src/tracing/spans/extractGenAiSpans.ts b/packages/core/src/tracing/spans/extractGenAiSpans.ts new file mode 100644 index 000000000000..ffd9c6401506 --- /dev/null +++ b/packages/core/src/tracing/spans/extractGenAiSpans.ts @@ -0,0 +1,46 @@ +import type { Client } from '../../client'; +import type { SpanContainerItem } from '../../types-hoist/envelope'; +import type { Event } from '../../types-hoist/event'; +import { hasSpanStreamingEnabled } from './hasSpanStreamingEnabled'; +import { spanJsonToSerializedStreamedSpan } from './spanJsonToStreamedSpan'; + +/** + * Extracts gen_ai spans from a transaction event, converts them to span v2 format, + * and returns them as a SpanContainerItem. + * + * Only applies to static mode (non-streaming) transactions. + * + * WARNING: This function mutates `event.spans` by removing the extracted gen_ai spans + * from the array. Call this before creating the event envelope so the transaction + * item does not include the extracted spans. + */ +export function extractGenAiSpansFromEvent(event: Event, client: Client): SpanContainerItem | undefined { + if (event.type !== 'transaction' || !event.spans?.length || hasSpanStreamingEnabled(client)) { + return undefined; + } + + const genAiSpans = []; + const remainingSpans = []; + + for (const span of event.spans) { + if (span.op?.startsWith('gen_ai.')) { + genAiSpans.push(span); + } else { + remainingSpans.push(span); + } + } + + if (genAiSpans.length === 0) { + return undefined; + } + + const serializedSpans = genAiSpans.map(span => spanJsonToSerializedStreamedSpan(span, event, client)); + + // Remove gen_ai spans from the legacy transaction + event.spans = remainingSpans; + + return [ + { type: 'span', item_count: serializedSpans.length, content_type: 'application/vnd.sentry.items.span.v2+json' }, + { items: serializedSpans }, + ]; +} diff --git a/packages/core/src/tracing/spans/spanJsonToStreamedSpan.ts b/packages/core/src/tracing/spans/spanJsonToStreamedSpan.ts new file mode 100644 index 000000000000..342f8217a5ca --- /dev/null +++ b/packages/core/src/tracing/spans/spanJsonToStreamedSpan.ts @@ -0,0 +1,78 @@ +import type { RawAttributes } from '../../attributes'; +import type { Client } from '../../client'; +import { + SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT, + SEMANTIC_ATTRIBUTE_SENTRY_OP, + SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, + SEMANTIC_ATTRIBUTE_SENTRY_RELEASE, + SEMANTIC_ATTRIBUTE_SENTRY_SDK_NAME, + SEMANTIC_ATTRIBUTE_SENTRY_SDK_VERSION, + SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_ID, + SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_NAME, + SEMANTIC_ATTRIBUTE_USER_EMAIL, + SEMANTIC_ATTRIBUTE_USER_ID, + SEMANTIC_ATTRIBUTE_USER_IP_ADDRESS, + SEMANTIC_ATTRIBUTE_USER_USERNAME, +} from '../../semanticAttributes'; +import type { Event } from '../../types-hoist/event'; +import type { SerializedStreamedSpan, SpanJSON, StreamedSpanJSON } from '../../types-hoist/span'; +import { streamedSpanJsonToSerializedSpan } from '../../utils/spanUtils'; +import { safeSetSpanJSONAttributes } from './captureSpan'; + +/** + * Converts a v1 SpanJSON (from a legacy transaction) to a serialized v2 StreamedSpan. + */ +export function spanJsonToSerializedStreamedSpan( + span: SpanJSON, + transactionEvent: Event, + client: Client, +): SerializedStreamedSpan { + const streamedSpan: StreamedSpanJSON = { + trace_id: span.trace_id, + span_id: span.span_id, + parent_span_id: span.parent_span_id, + name: span.description || '', + start_timestamp: span.start_timestamp, + end_timestamp: span.timestamp || span.start_timestamp, + status: mapV1StatusToV2(span.status), + is_segment: false, + attributes: { ...(span.data as RawAttributes>) }, + links: span.links, + }; + + // Fold op and origin into attributes + safeSetSpanJSONAttributes(streamedSpan, { + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: span.op, + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: span.origin, + }); + + // Enrich from transaction event context (same pattern as captureSpan.ts applyCommonSpanAttributes) + const sdk = client.getSdkMetadata(); + const { release, environment, sendDefaultPii } = client.getOptions(); + + safeSetSpanJSONAttributes(streamedSpan, { + [SEMANTIC_ATTRIBUTE_SENTRY_RELEASE]: transactionEvent.release || release, + [SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT]: transactionEvent.environment || environment, + [SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_NAME]: transactionEvent.transaction, + [SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_ID]: transactionEvent.contexts?.trace?.span_id, + [SEMANTIC_ATTRIBUTE_SENTRY_SDK_NAME]: sdk?.sdk?.name, + [SEMANTIC_ATTRIBUTE_SENTRY_SDK_VERSION]: sdk?.sdk?.version, + ...(sendDefaultPii + ? { + [SEMANTIC_ATTRIBUTE_USER_ID]: transactionEvent.user?.id, + [SEMANTIC_ATTRIBUTE_USER_EMAIL]: transactionEvent.user?.email, + [SEMANTIC_ATTRIBUTE_USER_IP_ADDRESS]: transactionEvent.user?.ip_address, + [SEMANTIC_ATTRIBUTE_USER_USERNAME]: transactionEvent.user?.username, + } + : {}), + }); + + return streamedSpanJsonToSerializedSpan(streamedSpan); +} + +function mapV1StatusToV2(status: string | undefined): 'ok' | 'error' { + if (!status || status === 'ok' || status === 'cancelled') { + return 'ok'; + } + return 'error'; +} diff --git a/packages/core/src/types-hoist/envelope.ts b/packages/core/src/types-hoist/envelope.ts index d8b8a1822b04..8937c699a333 100644 --- a/packages/core/src/types-hoist/envelope.ts +++ b/packages/core/src/types-hoist/envelope.ts @@ -154,7 +154,7 @@ type LogEnvelopeHeaders = BaseEnvelopeHeaders; type MetricEnvelopeHeaders = BaseEnvelopeHeaders; export type EventEnvelope = BaseEnvelope< EventEnvelopeHeaders, - EventItem | AttachmentItem | UserFeedbackItem | FeedbackItem | ProfileItem + EventItem | AttachmentItem | UserFeedbackItem | FeedbackItem | ProfileItem | SpanContainerItem >; export type SessionEnvelope = BaseEnvelope; export type ClientReportEnvelope = BaseEnvelope; diff --git a/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts b/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts new file mode 100644 index 000000000000..911c05f589d2 --- /dev/null +++ b/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from 'vitest'; +import type { Event } from '../../../../src/types-hoist/event'; +import type { SpanJSON } from '../../../../src/types-hoist/span'; +import { extractGenAiSpansFromEvent } from '../../../../src/tracing/spans/extractGenAiSpans'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +function makeSpanJSON(overrides: Partial = {}): SpanJSON { + return { + span_id: 'abc123def456789a', + trace_id: '00112233445566778899aabbccddeeff', + start_timestamp: 1000, + data: {}, + ...overrides, + }; +} + +function makeTransactionEvent(spans: SpanJSON[]): Event { + return { + type: 'transaction', + transaction: 'GET /api/chat', + release: '1.0.0', + environment: 'production', + contexts: { + trace: { + span_id: 'root0000deadbeef', + trace_id: '00112233445566778899aabbccddeeff', + }, + }, + spans, + }; +} + +function makeClient(options: Partial[0]> = {}): TestClient { + return new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://dsn@ingest.f00.f00/1', + ...options, + }), + ); +} + +describe('extractGenAiSpansFromEvent', () => { + it('extracts gen_ai spans and removes them from the event', () => { + const genAiSpan = makeSpanJSON({ + span_id: 'genai001', + op: 'gen_ai.chat', + description: 'chat gpt-4', + timestamp: 1005, + }); + const httpSpan = makeSpanJSON({ + span_id: 'http001', + op: 'http.client', + description: 'GET /api', + timestamp: 1002, + }); + + const event = makeTransactionEvent([genAiSpan, httpSpan]); + const client = makeClient(); + + const result = extractGenAiSpansFromEvent(event, client); + + // gen_ai spans should be in the container item + expect(result).toBeDefined(); + const [headers, payload] = result!; + expect(headers.type).toBe('span'); + expect(headers.item_count).toBe(1); + expect(headers.content_type).toBe('application/vnd.sentry.items.span.v2+json'); + expect(payload.items).toHaveLength(1); + expect(payload.items[0]!.span_id).toBe('genai001'); + expect(payload.items[0]!.name).toBe('chat gpt-4'); + + // gen_ai spans should be removed from the event + expect(event.spans).toHaveLength(1); + expect(event.spans![0]!.span_id).toBe('http001'); + }); + + it('extracts multiple gen_ai spans', () => { + const chatSpan = makeSpanJSON({ span_id: 'chat001', op: 'gen_ai.chat', description: 'chat' }); + const embeddingsSpan = makeSpanJSON({ span_id: 'embed001', op: 'gen_ai.embeddings', description: 'embed' }); + const agentSpan = makeSpanJSON({ span_id: 'agent001', op: 'gen_ai.invoke_agent', description: 'agent' }); + const dbSpan = makeSpanJSON({ span_id: 'db001', op: 'db.query', description: 'SELECT *' }); + + const event = makeTransactionEvent([chatSpan, embeddingsSpan, dbSpan, agentSpan]); + const client = makeClient(); + + const result = extractGenAiSpansFromEvent(event, client); + + expect(result).toBeDefined(); + expect(result![0].item_count).toBe(3); + expect(result![1].items).toHaveLength(3); + expect(result![1].items.map(s => s.span_id)).toEqual(['chat001', 'embed001', 'agent001']); + + // Only the db span should remain + expect(event.spans).toHaveLength(1); + expect(event.spans![0]!.span_id).toBe('db001'); + }); + + it('returns undefined when there are no gen_ai spans', () => { + const httpSpan = makeSpanJSON({ op: 'http.client' }); + const dbSpan = makeSpanJSON({ op: 'db.query' }); + + const event = makeTransactionEvent([httpSpan, dbSpan]); + const client = makeClient(); + + const result = extractGenAiSpansFromEvent(event, client); + + expect(result).toBeUndefined(); + expect(event.spans).toHaveLength(2); + }); + + it('returns undefined when event has no spans', () => { + const event = makeTransactionEvent([]); + const client = makeClient(); + + expect(extractGenAiSpansFromEvent(event, client)).toBeUndefined(); + }); + + it('returns undefined when event is not a transaction', () => { + const event: Event = { type: undefined, spans: [makeSpanJSON({ op: 'gen_ai.chat' })] }; + const client = makeClient(); + + expect(extractGenAiSpansFromEvent(event, client)).toBeUndefined(); + }); + + it('returns undefined when span streaming is enabled', () => { + const event = makeTransactionEvent([makeSpanJSON({ op: 'gen_ai.chat' })]); + const client = makeClient({ traceLifecycle: 'stream' }); + + expect(extractGenAiSpansFromEvent(event, client)).toBeUndefined(); + // Spans should not be modified + expect(event.spans).toHaveLength(1); + }); + + it('preserves parent_span_id pointing to v1 spans', () => { + const genAiSpan = makeSpanJSON({ + span_id: 'genai001', + parent_span_id: 'http001', + op: 'gen_ai.chat', + }); + const httpSpan = makeSpanJSON({ + span_id: 'http001', + op: 'http.client', + }); + + const event = makeTransactionEvent([httpSpan, genAiSpan]); + const client = makeClient(); + + const result = extractGenAiSpansFromEvent(event, client); + + // The v2 span should still reference the v1 parent + expect(result![1].items[0]!.parent_span_id).toBe('http001'); + // The v1 parent should remain in the transaction + expect(event.spans).toHaveLength(1); + expect(event.spans![0]!.span_id).toBe('http001'); + }); +}); diff --git a/packages/core/test/lib/tracing/spans/spanJsonToStreamedSpan.test.ts b/packages/core/test/lib/tracing/spans/spanJsonToStreamedSpan.test.ts new file mode 100644 index 000000000000..08a3a0ae05f5 --- /dev/null +++ b/packages/core/test/lib/tracing/spans/spanJsonToStreamedSpan.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, it } from 'vitest'; +import { + SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT, + SEMANTIC_ATTRIBUTE_SENTRY_OP, + SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, + SEMANTIC_ATTRIBUTE_SENTRY_RELEASE, + SEMANTIC_ATTRIBUTE_SENTRY_SDK_NAME, + SEMANTIC_ATTRIBUTE_SENTRY_SDK_VERSION, + SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_ID, + SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_NAME, + SEMANTIC_ATTRIBUTE_USER_EMAIL, + SEMANTIC_ATTRIBUTE_USER_ID, + SEMANTIC_ATTRIBUTE_USER_IP_ADDRESS, + SEMANTIC_ATTRIBUTE_USER_USERNAME, +} from '../../../../src'; +import type { Event } from '../../../../src/types-hoist/event'; +import type { SpanJSON } from '../../../../src/types-hoist/span'; +import { spanJsonToSerializedStreamedSpan } from '../../../../src/tracing/spans/spanJsonToStreamedSpan'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +function makeSpanJSON(overrides: Partial = {}): SpanJSON { + return { + span_id: 'abc123def456789a', + trace_id: '00112233445566778899aabbccddeeff', + start_timestamp: 1000, + data: {}, + ...overrides, + }; +} + +function makeTransactionEvent(overrides: Partial = {}): Event { + return { + type: 'transaction', + transaction: 'GET /api/chat', + release: '1.0.0', + environment: 'production', + contexts: { + trace: { + span_id: 'root0000deadbeef', + trace_id: '00112233445566778899aabbccddeeff', + }, + }, + ...overrides, + }; +} + +function makeClient(options: Partial[0]> = {}): TestClient { + return new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://dsn@ingest.f00.f00/1', + ...options, + }), + ); +} + +describe('spanJsonToSerializedStreamedSpan', () => { + it('maps basic SpanJSON fields to StreamedSpan fields', () => { + const span = makeSpanJSON({ + description: 'chat gpt-4', + timestamp: 1005, + status: 'ok', + op: 'gen_ai.chat', + origin: 'auto.ai.openai', + parent_span_id: 'parent00deadbeef', + }); + + const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + + expect(result.name).toBe('chat gpt-4'); + expect(result.start_timestamp).toBe(1000); + expect(result.end_timestamp).toBe(1005); + expect(result.status).toBe('ok'); + expect(result.is_segment).toBe(false); + expect(result.span_id).toBe('abc123def456789a'); + expect(result.trace_id).toBe('00112233445566778899aabbccddeeff'); + expect(result.parent_span_id).toBe('parent00deadbeef'); + }); + + it('uses empty string for name when description is undefined', () => { + const span = makeSpanJSON({ description: undefined }); + const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + expect(result.name).toBe(''); + }); + + it('uses start_timestamp as end_timestamp when timestamp is undefined', () => { + const span = makeSpanJSON({ timestamp: undefined }); + const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + expect(result.end_timestamp).toBe(1000); + }); + + it('maps v1 status strings to v2 ok/error', () => { + const cases: Array<[string | undefined, 'ok' | 'error']> = [ + [undefined, 'ok'], + ['ok', 'ok'], + ['cancelled', 'ok'], + ['internal_error', 'error'], + ['not_found', 'error'], + ['unknown_error', 'error'], + ]; + + for (const [v1Status, expected] of cases) { + const span = makeSpanJSON({ status: v1Status }); + const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + expect(result.status).toBe(expected); + } + }); + + it('folds op and origin into attributes', () => { + const span = makeSpanJSON({ + op: 'gen_ai.chat', + origin: 'auto.ai.openai', + }); + + const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ type: 'string', value: 'auto.ai.openai' }); + }); + + it('preserves existing span data attributes', () => { + const span = makeSpanJSON({ + data: { + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.usage.input_tokens': 100, + 'gen_ai.usage.output_tokens': 50, + }, + }); + + const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + + expect(result.attributes?.['gen_ai.system']).toEqual({ type: 'string', value: 'openai' }); + expect(result.attributes?.['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); + expect(result.attributes?.['gen_ai.usage.input_tokens']).toEqual({ type: 'integer', value: 100 }); + expect(result.attributes?.['gen_ai.usage.output_tokens']).toEqual({ type: 'integer', value: 50 }); + }); + + it('enriches with transaction event context', () => { + const span = makeSpanJSON(); + const event = makeTransactionEvent({ + release: '2.0.0', + environment: 'staging', + transaction: 'POST /api/generate', + contexts: { trace: { span_id: 'segment0deadbeef' } }, + }); + + const result = spanJsonToSerializedStreamedSpan(span, event, makeClient()); + + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_RELEASE]).toEqual({ type: 'string', value: '2.0.0' }); + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT]).toEqual({ type: 'string', value: 'staging' }); + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_NAME]).toEqual({ + type: 'string', + value: 'POST /api/generate', + }); + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_ID]).toEqual({ + type: 'string', + value: 'segment0deadbeef', + }); + }); + + it('enriches with SDK metadata', () => { + const client = makeClient(); + client.getOptions()._metadata = { sdk: { name: 'sentry.javascript.node', version: '9.0.0' } }; + + const span = makeSpanJSON(); + const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), client); + + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_SDK_NAME]).toEqual({ + type: 'string', + value: 'sentry.javascript.node', + }); + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_SDK_VERSION]).toEqual({ type: 'string', value: '9.0.0' }); + }); + + it('adds user attributes when sendDefaultPii is true', () => { + const client = makeClient({ sendDefaultPii: true }); + const event = makeTransactionEvent({ + user: { id: 'u123', email: 'a@b.com', ip_address: '1.2.3.4', username: 'alice' }, + }); + + const span = makeSpanJSON(); + const result = spanJsonToSerializedStreamedSpan(span, event, client); + + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_ID]).toEqual({ type: 'string', value: 'u123' }); + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_EMAIL]).toEqual({ type: 'string', value: 'a@b.com' }); + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_IP_ADDRESS]).toEqual({ type: 'string', value: '1.2.3.4' }); + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_USERNAME]).toEqual({ type: 'string', value: 'alice' }); + }); + + it('does not add user attributes when sendDefaultPii is false', () => { + const client = makeClient({ sendDefaultPii: false }); + const event = makeTransactionEvent({ + user: { id: 'u123', email: 'a@b.com' }, + }); + + const span = makeSpanJSON(); + const result = spanJsonToSerializedStreamedSpan(span, event, client); + + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_ID]).toBeUndefined(); + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_EMAIL]).toBeUndefined(); + }); + + it('does not overwrite pre-existing span data attributes with enrichment', () => { + const span = makeSpanJSON({ + data: { + [SEMANTIC_ATTRIBUTE_SENTRY_RELEASE]: 'span-level-release', + [SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT]: 'span-level-env', + }, + }); + + const event = makeTransactionEvent({ + release: 'event-level-release', + environment: 'event-level-env', + }); + + const result = spanJsonToSerializedStreamedSpan(span, event, makeClient()); + + // Span-level values should win + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_RELEASE]).toEqual({ + type: 'string', + value: 'span-level-release', + }); + expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT]).toEqual({ + type: 'string', + value: 'span-level-env', + }); + }); + + it('carries over links', () => { + const span = makeSpanJSON({ + links: [{ trace_id: 'aabb', span_id: 'ccdd', sampled: true, attributes: { foo: 'bar' } }], + }); + + const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + + expect(result.links).toEqual([ + { trace_id: 'aabb', span_id: 'ccdd', sampled: true, attributes: { foo: { type: 'string', value: 'bar' } } }, + ]); + }); +}); From 69b5cf818eb5f7803a19089509445af1caa8fed1 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 16 Apr 2026 17:59:22 +0900 Subject: [PATCH 02/18] Remove SDK-side enrichment and redundant op/origin backfill --- .../src/tracing/spans/extractGenAiSpans.ts | 2 +- .../tracing/spans/spanJsonToStreamedSpan.ts | 59 +------ .../spans/spanJsonToStreamedSpan.test.ts | 159 +----------------- 3 files changed, 9 insertions(+), 211 deletions(-) diff --git a/packages/core/src/tracing/spans/extractGenAiSpans.ts b/packages/core/src/tracing/spans/extractGenAiSpans.ts index ffd9c6401506..be86f9fa62f4 100644 --- a/packages/core/src/tracing/spans/extractGenAiSpans.ts +++ b/packages/core/src/tracing/spans/extractGenAiSpans.ts @@ -34,7 +34,7 @@ export function extractGenAiSpansFromEvent(event: Event, client: Client): SpanCo return undefined; } - const serializedSpans = genAiSpans.map(span => spanJsonToSerializedStreamedSpan(span, event, client)); + const serializedSpans = genAiSpans.map(span => spanJsonToSerializedStreamedSpan(span)); // Remove gen_ai spans from the legacy transaction event.spans = remainingSpans; diff --git a/packages/core/src/tracing/spans/spanJsonToStreamedSpan.ts b/packages/core/src/tracing/spans/spanJsonToStreamedSpan.ts index 342f8217a5ca..4dfd6c5202b9 100644 --- a/packages/core/src/tracing/spans/spanJsonToStreamedSpan.ts +++ b/packages/core/src/tracing/spans/spanJsonToStreamedSpan.ts @@ -1,32 +1,11 @@ import type { RawAttributes } from '../../attributes'; -import type { Client } from '../../client'; -import { - SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT, - SEMANTIC_ATTRIBUTE_SENTRY_OP, - SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, - SEMANTIC_ATTRIBUTE_SENTRY_RELEASE, - SEMANTIC_ATTRIBUTE_SENTRY_SDK_NAME, - SEMANTIC_ATTRIBUTE_SENTRY_SDK_VERSION, - SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_ID, - SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_NAME, - SEMANTIC_ATTRIBUTE_USER_EMAIL, - SEMANTIC_ATTRIBUTE_USER_ID, - SEMANTIC_ATTRIBUTE_USER_IP_ADDRESS, - SEMANTIC_ATTRIBUTE_USER_USERNAME, -} from '../../semanticAttributes'; -import type { Event } from '../../types-hoist/event'; import type { SerializedStreamedSpan, SpanJSON, StreamedSpanJSON } from '../../types-hoist/span'; import { streamedSpanJsonToSerializedSpan } from '../../utils/spanUtils'; -import { safeSetSpanJSONAttributes } from './captureSpan'; /** * Converts a v1 SpanJSON (from a legacy transaction) to a serialized v2 StreamedSpan. */ -export function spanJsonToSerializedStreamedSpan( - span: SpanJSON, - transactionEvent: Event, - client: Client, -): SerializedStreamedSpan { +export function spanJsonToSerializedStreamedSpan(span: SpanJSON): SerializedStreamedSpan { const streamedSpan: StreamedSpanJSON = { trace_id: span.trace_id, span_id: span.span_id, @@ -34,45 +13,11 @@ export function spanJsonToSerializedStreamedSpan( name: span.description || '', start_timestamp: span.start_timestamp, end_timestamp: span.timestamp || span.start_timestamp, - status: mapV1StatusToV2(span.status), + status: !span.status || span.status === 'ok' || span.status === 'cancelled' ? 'ok' : 'error', is_segment: false, attributes: { ...(span.data as RawAttributes>) }, links: span.links, }; - // Fold op and origin into attributes - safeSetSpanJSONAttributes(streamedSpan, { - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: span.op, - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: span.origin, - }); - - // Enrich from transaction event context (same pattern as captureSpan.ts applyCommonSpanAttributes) - const sdk = client.getSdkMetadata(); - const { release, environment, sendDefaultPii } = client.getOptions(); - - safeSetSpanJSONAttributes(streamedSpan, { - [SEMANTIC_ATTRIBUTE_SENTRY_RELEASE]: transactionEvent.release || release, - [SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT]: transactionEvent.environment || environment, - [SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_NAME]: transactionEvent.transaction, - [SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_ID]: transactionEvent.contexts?.trace?.span_id, - [SEMANTIC_ATTRIBUTE_SENTRY_SDK_NAME]: sdk?.sdk?.name, - [SEMANTIC_ATTRIBUTE_SENTRY_SDK_VERSION]: sdk?.sdk?.version, - ...(sendDefaultPii - ? { - [SEMANTIC_ATTRIBUTE_USER_ID]: transactionEvent.user?.id, - [SEMANTIC_ATTRIBUTE_USER_EMAIL]: transactionEvent.user?.email, - [SEMANTIC_ATTRIBUTE_USER_IP_ADDRESS]: transactionEvent.user?.ip_address, - [SEMANTIC_ATTRIBUTE_USER_USERNAME]: transactionEvent.user?.username, - } - : {}), - }); - return streamedSpanJsonToSerializedSpan(streamedSpan); } - -function mapV1StatusToV2(status: string | undefined): 'ok' | 'error' { - if (!status || status === 'ok' || status === 'cancelled') { - return 'ok'; - } - return 'error'; -} diff --git a/packages/core/test/lib/tracing/spans/spanJsonToStreamedSpan.test.ts b/packages/core/test/lib/tracing/spans/spanJsonToStreamedSpan.test.ts index 08a3a0ae05f5..a42ce3468e65 100644 --- a/packages/core/test/lib/tracing/spans/spanJsonToStreamedSpan.test.ts +++ b/packages/core/test/lib/tracing/spans/spanJsonToStreamedSpan.test.ts @@ -1,22 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { - SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT, - SEMANTIC_ATTRIBUTE_SENTRY_OP, - SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, - SEMANTIC_ATTRIBUTE_SENTRY_RELEASE, - SEMANTIC_ATTRIBUTE_SENTRY_SDK_NAME, - SEMANTIC_ATTRIBUTE_SENTRY_SDK_VERSION, - SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_ID, - SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_NAME, - SEMANTIC_ATTRIBUTE_USER_EMAIL, - SEMANTIC_ATTRIBUTE_USER_ID, - SEMANTIC_ATTRIBUTE_USER_IP_ADDRESS, - SEMANTIC_ATTRIBUTE_USER_USERNAME, -} from '../../../../src'; -import type { Event } from '../../../../src/types-hoist/event'; import type { SpanJSON } from '../../../../src/types-hoist/span'; import { spanJsonToSerializedStreamedSpan } from '../../../../src/tracing/spans/spanJsonToStreamedSpan'; -import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; function makeSpanJSON(overrides: Partial = {}): SpanJSON { return { @@ -28,31 +12,6 @@ function makeSpanJSON(overrides: Partial = {}): SpanJSON { }; } -function makeTransactionEvent(overrides: Partial = {}): Event { - return { - type: 'transaction', - transaction: 'GET /api/chat', - release: '1.0.0', - environment: 'production', - contexts: { - trace: { - span_id: 'root0000deadbeef', - trace_id: '00112233445566778899aabbccddeeff', - }, - }, - ...overrides, - }; -} - -function makeClient(options: Partial[0]> = {}): TestClient { - return new TestClient( - getDefaultTestClientOptions({ - dsn: 'https://dsn@ingest.f00.f00/1', - ...options, - }), - ); -} - describe('spanJsonToSerializedStreamedSpan', () => { it('maps basic SpanJSON fields to StreamedSpan fields', () => { const span = makeSpanJSON({ @@ -64,7 +23,7 @@ describe('spanJsonToSerializedStreamedSpan', () => { parent_span_id: 'parent00deadbeef', }); - const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + const result = spanJsonToSerializedStreamedSpan(span); expect(result.name).toBe('chat gpt-4'); expect(result.start_timestamp).toBe(1000); @@ -77,14 +36,12 @@ describe('spanJsonToSerializedStreamedSpan', () => { }); it('uses empty string for name when description is undefined', () => { - const span = makeSpanJSON({ description: undefined }); - const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + const result = spanJsonToSerializedStreamedSpan(makeSpanJSON({ description: undefined })); expect(result.name).toBe(''); }); it('uses start_timestamp as end_timestamp when timestamp is undefined', () => { - const span = makeSpanJSON({ timestamp: undefined }); - const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + const result = spanJsonToSerializedStreamedSpan(makeSpanJSON({ timestamp: undefined })); expect(result.end_timestamp).toBe(1000); }); @@ -99,24 +56,11 @@ describe('spanJsonToSerializedStreamedSpan', () => { ]; for (const [v1Status, expected] of cases) { - const span = makeSpanJSON({ status: v1Status }); - const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + const result = spanJsonToSerializedStreamedSpan(makeSpanJSON({ status: v1Status })); expect(result.status).toBe(expected); } }); - it('folds op and origin into attributes', () => { - const span = makeSpanJSON({ - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - }); - - const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); - - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ type: 'string', value: 'auto.ai.openai' }); - }); - it('preserves existing span data attributes', () => { const span = makeSpanJSON({ data: { @@ -127,7 +71,7 @@ describe('spanJsonToSerializedStreamedSpan', () => { }, }); - const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + const result = spanJsonToSerializedStreamedSpan(span); expect(result.attributes?.['gen_ai.system']).toEqual({ type: 'string', value: 'openai' }); expect(result.attributes?.['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); @@ -135,103 +79,12 @@ describe('spanJsonToSerializedStreamedSpan', () => { expect(result.attributes?.['gen_ai.usage.output_tokens']).toEqual({ type: 'integer', value: 50 }); }); - it('enriches with transaction event context', () => { - const span = makeSpanJSON(); - const event = makeTransactionEvent({ - release: '2.0.0', - environment: 'staging', - transaction: 'POST /api/generate', - contexts: { trace: { span_id: 'segment0deadbeef' } }, - }); - - const result = spanJsonToSerializedStreamedSpan(span, event, makeClient()); - - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_RELEASE]).toEqual({ type: 'string', value: '2.0.0' }); - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT]).toEqual({ type: 'string', value: 'staging' }); - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_NAME]).toEqual({ - type: 'string', - value: 'POST /api/generate', - }); - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_SEGMENT_ID]).toEqual({ - type: 'string', - value: 'segment0deadbeef', - }); - }); - - it('enriches with SDK metadata', () => { - const client = makeClient(); - client.getOptions()._metadata = { sdk: { name: 'sentry.javascript.node', version: '9.0.0' } }; - - const span = makeSpanJSON(); - const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), client); - - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_SDK_NAME]).toEqual({ - type: 'string', - value: 'sentry.javascript.node', - }); - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_SDK_VERSION]).toEqual({ type: 'string', value: '9.0.0' }); - }); - - it('adds user attributes when sendDefaultPii is true', () => { - const client = makeClient({ sendDefaultPii: true }); - const event = makeTransactionEvent({ - user: { id: 'u123', email: 'a@b.com', ip_address: '1.2.3.4', username: 'alice' }, - }); - - const span = makeSpanJSON(); - const result = spanJsonToSerializedStreamedSpan(span, event, client); - - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_ID]).toEqual({ type: 'string', value: 'u123' }); - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_EMAIL]).toEqual({ type: 'string', value: 'a@b.com' }); - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_IP_ADDRESS]).toEqual({ type: 'string', value: '1.2.3.4' }); - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_USERNAME]).toEqual({ type: 'string', value: 'alice' }); - }); - - it('does not add user attributes when sendDefaultPii is false', () => { - const client = makeClient({ sendDefaultPii: false }); - const event = makeTransactionEvent({ - user: { id: 'u123', email: 'a@b.com' }, - }); - - const span = makeSpanJSON(); - const result = spanJsonToSerializedStreamedSpan(span, event, client); - - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_ID]).toBeUndefined(); - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_USER_EMAIL]).toBeUndefined(); - }); - - it('does not overwrite pre-existing span data attributes with enrichment', () => { - const span = makeSpanJSON({ - data: { - [SEMANTIC_ATTRIBUTE_SENTRY_RELEASE]: 'span-level-release', - [SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT]: 'span-level-env', - }, - }); - - const event = makeTransactionEvent({ - release: 'event-level-release', - environment: 'event-level-env', - }); - - const result = spanJsonToSerializedStreamedSpan(span, event, makeClient()); - - // Span-level values should win - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_RELEASE]).toEqual({ - type: 'string', - value: 'span-level-release', - }); - expect(result.attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_ENVIRONMENT]).toEqual({ - type: 'string', - value: 'span-level-env', - }); - }); - it('carries over links', () => { const span = makeSpanJSON({ links: [{ trace_id: 'aabb', span_id: 'ccdd', sampled: true, attributes: { foo: 'bar' } }], }); - const result = spanJsonToSerializedStreamedSpan(span, makeTransactionEvent(), makeClient()); + const result = spanJsonToSerializedStreamedSpan(span); expect(result.links).toEqual([ { trace_id: 'aabb', span_id: 'ccdd', sampled: true, attributes: { foo: { type: 'string', value: 'bar' } } }, From 257c9fbffae4848861c26d8f9ab34ffefc3350a4 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 16 Apr 2026 19:38:24 +0900 Subject: [PATCH 03/18] Pre-check wether transactions have gen_ai spans when we construct transactions to lessen the extraction performance impact --- packages/core/src/tracing/sentrySpan.ts | 5 ++ .../src/tracing/spans/extractGenAiSpans.ts | 7 ++- .../core/test/lib/tracing/sentrySpan.test.ts | 1 + .../tracing/spans/extractGenAiSpans.test.ts | 54 ++++++++----------- packages/opentelemetry/src/spanExporter.ts | 30 +++++++++-- 5 files changed, 58 insertions(+), 39 deletions(-) diff --git a/packages/core/src/tracing/sentrySpan.ts b/packages/core/src/tracing/sentrySpan.ts index d9ab115b94cd..92c4617f29c2 100644 --- a/packages/core/src/tracing/sentrySpan.ts +++ b/packages/core/src/tracing/sentrySpan.ts @@ -392,8 +392,12 @@ export class SentrySpan implements Span { // remove internal root span attributes we don't need to send. /* eslint-disable @typescript-eslint/no-dynamic-delete */ delete this._attributes[SEMANTIC_ATTRIBUTE_SENTRY_CUSTOM_SPAN_NAME]; + let hasGenAiSpans = false; spans.forEach(span => { delete span.data[SEMANTIC_ATTRIBUTE_SENTRY_CUSTOM_SPAN_NAME]; + if (span.op?.startsWith('gen_ai.')) { + hasGenAiSpans = true; + } }); // eslint-enabled-next-line @typescript-eslint/no-dynamic-delete @@ -415,6 +419,7 @@ export class SentrySpan implements Span { capturedSpanScope, capturedSpanIsolationScope, dynamicSamplingContext: getDynamicSamplingContextFromSpan(this), + hasGenAiSpans, }, request: normalizedRequest, ...(source && { diff --git a/packages/core/src/tracing/spans/extractGenAiSpans.ts b/packages/core/src/tracing/spans/extractGenAiSpans.ts index be86f9fa62f4..05437cfb5802 100644 --- a/packages/core/src/tracing/spans/extractGenAiSpans.ts +++ b/packages/core/src/tracing/spans/extractGenAiSpans.ts @@ -15,7 +15,12 @@ import { spanJsonToSerializedStreamedSpan } from './spanJsonToStreamedSpan'; * item does not include the extracted spans. */ export function extractGenAiSpansFromEvent(event: Event, client: Client): SpanContainerItem | undefined { - if (event.type !== 'transaction' || !event.spans?.length || hasSpanStreamingEnabled(client)) { + if ( + event.type !== 'transaction' || + !event.spans?.length || + !event.sdkProcessingMetadata?.hasGenAiSpans || + hasSpanStreamingEnabled(client) + ) { return undefined; } diff --git a/packages/core/test/lib/tracing/sentrySpan.test.ts b/packages/core/test/lib/tracing/sentrySpan.test.ts index 4b70e1c3ef97..57ac2cdf5ba3 100644 --- a/packages/core/test/lib/tracing/sentrySpan.test.ts +++ b/packages/core/test/lib/tracing/sentrySpan.test.ts @@ -234,6 +234,7 @@ describe('SentrySpan', () => { trace_id: expect.stringMatching(/^[a-f0-9]{32}$/), transaction: 'test', }, + hasGenAiSpans: false, }, spans: [], start_timestamp: 1, diff --git a/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts b/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts index 911c05f589d2..c4a9290cf02c 100644 --- a/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts +++ b/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts @@ -14,7 +14,7 @@ function makeSpanJSON(overrides: Partial = {}): SpanJSON { }; } -function makeTransactionEvent(spans: SpanJSON[]): Event { +function makeTransactionEvent(spans: SpanJSON[], hasGenAiSpans = false): Event { return { type: 'transaction', transaction: 'GET /api/chat', @@ -26,6 +26,9 @@ function makeTransactionEvent(spans: SpanJSON[]): Event { trace_id: '00112233445566778899aabbccddeeff', }, }, + sdkProcessingMetadata: { + ...(hasGenAiSpans && { hasGenAiSpans: true }), + }, spans, }; } @@ -54,12 +57,9 @@ describe('extractGenAiSpansFromEvent', () => { timestamp: 1002, }); - const event = makeTransactionEvent([genAiSpan, httpSpan]); - const client = makeClient(); - - const result = extractGenAiSpansFromEvent(event, client); + const event = makeTransactionEvent([genAiSpan, httpSpan], true); + const result = extractGenAiSpansFromEvent(event, makeClient()); - // gen_ai spans should be in the container item expect(result).toBeDefined(); const [headers, payload] = result!; expect(headers.type).toBe('span'); @@ -69,7 +69,6 @@ describe('extractGenAiSpansFromEvent', () => { expect(payload.items[0]!.span_id).toBe('genai001'); expect(payload.items[0]!.name).toBe('chat gpt-4'); - // gen_ai spans should be removed from the event expect(event.spans).toHaveLength(1); expect(event.spans![0]!.span_id).toBe('http001'); }); @@ -80,54 +79,47 @@ describe('extractGenAiSpansFromEvent', () => { const agentSpan = makeSpanJSON({ span_id: 'agent001', op: 'gen_ai.invoke_agent', description: 'agent' }); const dbSpan = makeSpanJSON({ span_id: 'db001', op: 'db.query', description: 'SELECT *' }); - const event = makeTransactionEvent([chatSpan, embeddingsSpan, dbSpan, agentSpan]); - const client = makeClient(); - - const result = extractGenAiSpansFromEvent(event, client); + const event = makeTransactionEvent([chatSpan, embeddingsSpan, dbSpan, agentSpan], true); + const result = extractGenAiSpansFromEvent(event, makeClient()); expect(result).toBeDefined(); expect(result![0].item_count).toBe(3); expect(result![1].items).toHaveLength(3); expect(result![1].items.map(s => s.span_id)).toEqual(['chat001', 'embed001', 'agent001']); - // Only the db span should remain expect(event.spans).toHaveLength(1); expect(event.spans![0]!.span_id).toBe('db001'); }); - it('returns undefined when there are no gen_ai spans', () => { - const httpSpan = makeSpanJSON({ op: 'http.client' }); - const dbSpan = makeSpanJSON({ op: 'db.query' }); + it('returns undefined when hasGenAiSpans flag is not set', () => { + const event = makeTransactionEvent([makeSpanJSON({ op: 'gen_ai.chat' })], false); - const event = makeTransactionEvent([httpSpan, dbSpan]); - const client = makeClient(); + expect(extractGenAiSpansFromEvent(event, makeClient())).toBeUndefined(); + expect(event.spans).toHaveLength(1); + }); - const result = extractGenAiSpansFromEvent(event, client); + it('returns undefined when there are no gen_ai spans', () => { + const event = makeTransactionEvent([makeSpanJSON({ op: 'http.client' }), makeSpanJSON({ op: 'db.query' })], true); - expect(result).toBeUndefined(); + expect(extractGenAiSpansFromEvent(event, makeClient())).toBeUndefined(); expect(event.spans).toHaveLength(2); }); it('returns undefined when event has no spans', () => { const event = makeTransactionEvent([]); - const client = makeClient(); - - expect(extractGenAiSpansFromEvent(event, client)).toBeUndefined(); + expect(extractGenAiSpansFromEvent(event, makeClient())).toBeUndefined(); }); it('returns undefined when event is not a transaction', () => { const event: Event = { type: undefined, spans: [makeSpanJSON({ op: 'gen_ai.chat' })] }; - const client = makeClient(); - - expect(extractGenAiSpansFromEvent(event, client)).toBeUndefined(); + expect(extractGenAiSpansFromEvent(event, makeClient())).toBeUndefined(); }); it('returns undefined when span streaming is enabled', () => { - const event = makeTransactionEvent([makeSpanJSON({ op: 'gen_ai.chat' })]); + const event = makeTransactionEvent([makeSpanJSON({ op: 'gen_ai.chat' })], true); const client = makeClient({ traceLifecycle: 'stream' }); expect(extractGenAiSpansFromEvent(event, client)).toBeUndefined(); - // Spans should not be modified expect(event.spans).toHaveLength(1); }); @@ -142,14 +134,10 @@ describe('extractGenAiSpansFromEvent', () => { op: 'http.client', }); - const event = makeTransactionEvent([httpSpan, genAiSpan]); - const client = makeClient(); - - const result = extractGenAiSpansFromEvent(event, client); + const event = makeTransactionEvent([httpSpan, genAiSpan], true); + const result = extractGenAiSpansFromEvent(event, makeClient()); - // The v2 span should still reference the v1 parent expect(result![1].items[0]!.parent_span_id).toBe('http001'); - // The v1 parent should remain in the transaction expect(event.spans).toHaveLength(1); expect(event.spans![0]!.span_id).toBe('http001'); }); diff --git a/packages/opentelemetry/src/spanExporter.ts b/packages/opentelemetry/src/spanExporter.ts index f02df1d9d56c..aed57b52e58e 100644 --- a/packages/opentelemetry/src/spanExporter.ts +++ b/packages/opentelemetry/src/spanExporter.ts @@ -203,8 +203,11 @@ export class SentrySpanExporter { // We'll recursively add all the child spans to this array const spans = transactionEvent.spans || []; + let hasGenAiSpans = false; for (const child of root.children) { - createAndFinishSpanForOtelSpan(child, spans, sentSpans); + if (createAndFinishSpanForOtelSpan(child, spans, sentSpans)) { + hasGenAiSpans = true; + } } // spans.sort() mutates the array, but we do not use this anymore after this point @@ -214,6 +217,13 @@ export class SentrySpanExporter { ? spans.sort((a, b) => a.start_timestamp - b.start_timestamp).slice(0, MAX_SPAN_COUNT) : spans; + if (hasGenAiSpans) { + transactionEvent.sdkProcessingMetadata = { + ...transactionEvent.sdkProcessingMetadata, + hasGenAiSpans: true, + }; + } + const measurements = timedEventsToMeasurements(span.events); if (measurements) { transactionEvent.measurements = measurements; @@ -330,7 +340,10 @@ export function createTransactionForOtelSpan(span: ReadableSpan): TransactionEve return transactionEvent; } -function createAndFinishSpanForOtelSpan(node: SpanNode, spans: SpanJSON[], sentSpans: Set): void { +/** + * Returns `true` if this span or any descendant is a gen_ai span. + */ +function createAndFinishSpanForOtelSpan(node: SpanNode, spans: SpanJSON[], sentSpans: Set): boolean { const span = node.span; if (span) { @@ -341,10 +354,13 @@ function createAndFinishSpanForOtelSpan(node: SpanNode, spans: SpanJSON[], sentS // If this span should be dropped, we still want to create spans for the children of this if (shouldDrop) { + let hasGenAiSpans = false; node.children.forEach(child => { - createAndFinishSpanForOtelSpan(child, spans, sentSpans); + if (createAndFinishSpanForOtelSpan(child, spans, sentSpans)) { + hasGenAiSpans = true; + } }); - return; + return hasGenAiSpans; } const span_id = span.spanContext().spanId; @@ -381,9 +397,13 @@ function createAndFinishSpanForOtelSpan(node: SpanNode, spans: SpanJSON[], sentS spans.push(spanJSON); + let hasGenAiSpans = !!op?.startsWith('gen_ai.'); node.children.forEach(child => { - createAndFinishSpanForOtelSpan(child, spans, sentSpans); + if (createAndFinishSpanForOtelSpan(child, spans, sentSpans)) { + hasGenAiSpans = true; + } }); + return hasGenAiSpans; } function getSpanData(span: ReadableSpan): { From eeb78129c7a520ec6856d0bb9295e26c843b1fbb Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 16 Apr 2026 22:42:59 +0900 Subject: [PATCH 04/18] Inline span conversion --- .../core/src/tracing/spans/extractGenAiSpans.ts | 9 +++------ .../lib/tracing/spans/extractGenAiSpans.test.ts | 16 ++++++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/core/src/tracing/spans/extractGenAiSpans.ts b/packages/core/src/tracing/spans/extractGenAiSpans.ts index 05437cfb5802..ace5a6c9cb78 100644 --- a/packages/core/src/tracing/spans/extractGenAiSpans.ts +++ b/packages/core/src/tracing/spans/extractGenAiSpans.ts @@ -29,7 +29,7 @@ export function extractGenAiSpansFromEvent(event: Event, client: Client): SpanCo for (const span of event.spans) { if (span.op?.startsWith('gen_ai.')) { - genAiSpans.push(span); + genAiSpans.push(spanJsonToSerializedStreamedSpan(span)); } else { remainingSpans.push(span); } @@ -39,13 +39,10 @@ export function extractGenAiSpansFromEvent(event: Event, client: Client): SpanCo return undefined; } - const serializedSpans = genAiSpans.map(span => spanJsonToSerializedStreamedSpan(span)); - - // Remove gen_ai spans from the legacy transaction event.spans = remainingSpans; return [ - { type: 'span', item_count: serializedSpans.length, content_type: 'application/vnd.sentry.items.span.v2+json' }, - { items: serializedSpans }, + { type: 'span', item_count: genAiSpans.length, content_type: 'application/vnd.sentry.items.span.v2+json' }, + { items: genAiSpans }, ]; } diff --git a/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts b/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts index c4a9290cf02c..0ab6881b9ce4 100644 --- a/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts +++ b/packages/core/test/lib/tracing/spans/extractGenAiSpans.test.ts @@ -14,7 +14,7 @@ function makeSpanJSON(overrides: Partial = {}): SpanJSON { }; } -function makeTransactionEvent(spans: SpanJSON[], hasGenAiSpans = false): Event { +function makeTransactionEvent(spans: SpanJSON[]): Event { return { type: 'transaction', transaction: 'GET /api/chat', @@ -27,7 +27,7 @@ function makeTransactionEvent(spans: SpanJSON[], hasGenAiSpans = false): Event { }, }, sdkProcessingMetadata: { - ...(hasGenAiSpans && { hasGenAiSpans: true }), + hasGenAiSpans: true, }, spans, }; @@ -92,14 +92,18 @@ describe('extractGenAiSpansFromEvent', () => { }); it('returns undefined when hasGenAiSpans flag is not set', () => { - const event = makeTransactionEvent([makeSpanJSON({ op: 'gen_ai.chat' })], false); + const event: Event = { + type: 'transaction', + spans: [makeSpanJSON({ op: 'gen_ai.chat' })], + sdkProcessingMetadata: {}, + }; expect(extractGenAiSpansFromEvent(event, makeClient())).toBeUndefined(); expect(event.spans).toHaveLength(1); }); it('returns undefined when there are no gen_ai spans', () => { - const event = makeTransactionEvent([makeSpanJSON({ op: 'http.client' }), makeSpanJSON({ op: 'db.query' })], true); + const event = makeTransactionEvent([makeSpanJSON({ op: 'http.client' }), makeSpanJSON({ op: 'db.query' })]); expect(extractGenAiSpansFromEvent(event, makeClient())).toBeUndefined(); expect(event.spans).toHaveLength(2); @@ -116,7 +120,7 @@ describe('extractGenAiSpansFromEvent', () => { }); it('returns undefined when span streaming is enabled', () => { - const event = makeTransactionEvent([makeSpanJSON({ op: 'gen_ai.chat' })], true); + const event = makeTransactionEvent([makeSpanJSON({ op: 'gen_ai.chat' })]); const client = makeClient({ traceLifecycle: 'stream' }); expect(extractGenAiSpansFromEvent(event, client)).toBeUndefined(); @@ -134,7 +138,7 @@ describe('extractGenAiSpansFromEvent', () => { op: 'http.client', }); - const event = makeTransactionEvent([httpSpan, genAiSpan], true); + const event = makeTransactionEvent([httpSpan, genAiSpan]); const result = extractGenAiSpansFromEvent(event, makeClient()); expect(result![1].items[0]!.parent_span_id).toBe('http001'); From dc21b4961070dd61df249d4500fa978cafb60d75 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 10:53:58 +0900 Subject: [PATCH 05/18] Stringify array attributes in Vercel AI integration for v2 serialization compat The v2 span attribute serializer currently drops array values. Stringifying arrays in processEndedVercelAiSpan keeps attributes like gen_ai.response.finish_reasons intact when the span is serialized to the v2 format. Can be removed once span streaming supports arrays natively. Co-Authored-By: Claude Opus 4.6 --- packages/core/src/tracing/vercel-ai/index.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index 55b53c362612..b993602fc35c 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -336,6 +336,15 @@ function processEndedVercelAiSpan(span: SpanJSON): void { renameAttributeKey(attributes, key, `vercel.${key}`); } } + + // JSON-stringify any array-valued attributes so they survive v2 span serialization. + // The v2 serializer currently drops array values. Can be removed once span streaming + // supports arrays natively. + for (const [key, value] of Object.entries(attributes)) { + if (Array.isArray(value)) { + attributes[key] = JSON.stringify(value); + } + } } /** From d97c7f255356932197e65f45684ef7671ae3b396 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 10:53:58 +0900 Subject: [PATCH 06/18] Update Vercel AI Node tests Co-Authored-By: Claude Opus 4.6 --- .../tracing/vercelai/test-generate-object.ts | 91 +- .../suites/tracing/vercelai/test.ts | 1212 ++++++----------- .../suites/tracing/vercelai/v5/test.ts | 671 +++------ .../suites/tracing/vercelai/v6/test.ts | 777 +++-------- 4 files changed, 819 insertions(+), 1932 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test-generate-object.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test-generate-object.ts index 39e13d5425c2..54c64bc2172b 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test-generate-object.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test-generate-object.ts @@ -6,62 +6,45 @@ describe('Vercel AI integration - generateObject', () => { cleanupChildProcesses(); }); - const EXPECTED_TRANSACTION = { - transaction: 'main', - spans: expect.arrayContaining([ - // generateObject span - expect.objectContaining({ - data: expect.objectContaining({ - 'vercel.ai.model.id': 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateObject', - 'vercel.ai.pipeline.name': 'generateObject', - 'vercel.ai.streaming': false, - 'vercel.ai.settings.mode': 'json', - 'vercel.ai.settings.output': 'object', - 'gen_ai.request.schema': expect.any(String), - 'gen_ai.response.model': 'mock-model-id', - 'gen_ai.usage.input_tokens': 15, - 'gen_ai.usage.output_tokens': 25, - 'gen_ai.usage.total_tokens': 40, - 'gen_ai.operation.name': 'invoke_agent', - 'sentry.op': 'gen_ai.invoke_agent', - 'sentry.origin': 'auto.vercelai.otel', - }), - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // generateObject.doGenerate span - expect.objectContaining({ - data: expect.objectContaining({ - 'sentry.origin': 'auto.vercelai.otel', - 'sentry.op': 'gen_ai.generate_content', - 'gen_ai.operation.name': 'generate_content', - 'vercel.ai.operationId': 'ai.generateObject.doGenerate', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.model.id': 'mock-model-id', - 'vercel.ai.pipeline.name': 'generateObject.doGenerate', - 'vercel.ai.streaming': false, - 'gen_ai.system': 'mock-provider', - 'gen_ai.request.model': 'mock-model-id', - 'gen_ai.response.model': 'mock-model-id', - 'gen_ai.usage.input_tokens': 15, - 'gen_ai.usage.output_tokens': 25, - 'gen_ai.usage.total_tokens': 40, - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario-generate-object.mjs', 'instrument.mjs', (createRunner, test) => { test('captures generateObject spans with schema attributes', async () => { - await createRunner().expect({ transaction: EXPECTED_TRANSACTION }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] generateObject (invoke_agent) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.vercelai.otel'); + expect(firstSpan!.attributes['gen_ai.operation.name'].value).toBe('invoke_agent'); + expect(firstSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); + expect(firstSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); + expect(firstSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); + expect(firstSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); + expect(firstSpan!.attributes['gen_ai.request.schema']).toBeDefined(); + + // [1] generateObject.doGenerate (generate_content) + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject.doGenerate'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.vercelai.otel'); + expect(secondSpan!.attributes['gen_ai.operation.name'].value).toBe('generate_content'); + expect(secondSpan!.attributes['gen_ai.system'].value).toBe('mock-provider'); + expect(secondSpan!.attributes['gen_ai.request.model'].value).toBe('mock-model-id'); + expect(secondSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); + expect(secondSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); + expect(secondSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); + expect(secondSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); + }, + }) + .start() + .completed(); }); }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 5aa1dc8342a5..326117f1af6d 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -1,4 +1,3 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -9,8 +8,6 @@ import { GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE, - GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, - GEN_AI_RESPONSE_ID_ATTRIBUTE, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, GEN_AI_SYSTEM_ATTRIBUTE, GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE, @@ -31,576 +28,207 @@ describe('Vercel AI integration', () => { cleanupChildProcesses(); }); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - no telemetry config, should enable telemetry but not record inputs/outputs when sendDefaultPii: false - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.settings.maxSteps': 1, - 'vercel.ai.streaming': false, - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Second span - explicitly enabled telemetry but recordInputs/recordOutputs not set, should not record when sendDefaultPii: false - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Third span - explicit telemetry enabled, should record inputs/outputs regardless of sendDefaultPii - expect.objectContaining({ - data: { - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.settings.maxSteps': 1, - 'vercel.ai.streaming': false, - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Fourth span - doGenerate for explicit telemetry enabled call - expect.objectContaining({ - data: { - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.prompt.format': expect.any(String), - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Fifth span - tool call generateText span - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.settings.maxSteps': 1, - 'vercel.ai.streaming': false, - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Sixth span - tool call doGenerate span - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Seventh span - tool call execution span - // Note: gen_ai.tool.description is NOT present when sendDefaultPii: false because ai.prompt.tools is not recorded - expect.objectContaining({ - data: { - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.operationId': 'ai.toolCall', - }, - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - ]), - }; - - const EXPECTED_AVAILABLE_TOOLS_JSON = - '[{"type":"function","name":"getWeather","description":"Get the current weather for a location","parameters":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"],"additionalProperties":false,"$schema":"http://json-schema.org/draft-07/schema#"}}]'; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - no telemetry config, should enable telemetry AND record inputs/outputs when sendDefaultPii: true - expect.objectContaining({ - data: { - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the first span?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"Where is the first span?"}]', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.settings.maxSteps': 1, - 'vercel.ai.streaming': false, - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - parent_span_id: expect.any(String), - span_id: expect.any(String), - start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - trace_id: expect.any(String), - }), - // Second span - doGenerate for first call, should also include input/output fields when sendDefaultPii: true - expect.objectContaining({ - data: { - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"user","content":[{"type":"text","text":"Where is the first span?"}]}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.prompt.format': 'prompt', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - parent_span_id: expect.any(String), - span_id: expect.any(String), - start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - trace_id: expect.any(String), - }), - // Third span - explicitly enabled telemetry, should record inputs/outputs regardless of sendDefaultPii - expect.objectContaining({ - data: { - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.settings.maxSteps': 1, - 'vercel.ai.streaming': false, - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - parent_span_id: expect.any(String), - span_id: expect.any(String), - start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - trace_id: expect.any(String), - }), - // Fourth span - doGenerate for explicitly enabled telemetry call - expect.objectContaining({ - data: { - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.prompt.format': expect.any(String), - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - parent_span_id: expect.any(String), - span_id: expect.any(String), - start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - trace_id: expect.any(String), - }), - // Fifth span - tool call generateText span (should include prompts when sendDefaultPii: true) - expect.objectContaining({ - data: { - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather in San Francisco?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{ \\"location\\": \\"San Francisco\\" }"}],"finish_reason":"tool_call"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"What is the weather in San Francisco?"}]', - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.settings.maxSteps': 1, - 'vercel.ai.streaming': false, - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - parent_span_id: expect.any(String), - span_id: expect.any(String), - start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - trace_id: expect.any(String), - }), - // Sixth span - tool call doGenerate span (should include prompts when sendDefaultPii: true) - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: EXPECTED_AVAILABLE_TOOLS_JSON, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{ \\"location\\": \\"San Francisco\\" }"}],"finish_reason":"tool_call"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.prompt.format': expect.any(String), - 'vercel.ai.prompt.toolChoice': expect.any(String), - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - parent_span_id: expect.any(String), - span_id: expect.any(String), - start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - trace_id: expect.any(String), - }), - // Seventh span - tool call execution span - expect.objectContaining({ - data: { - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE]: 'Get the current weather for a location', - [GEN_AI_TOOL_INPUT_ATTRIBUTE]: expect.any(String), - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_OUTPUT_ATTRIBUTE]: expect.any(String), - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.operationId': 'ai.toolCall', - }, - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'ok', - parent_span_id: expect.any(String), - span_id: expect.any(String), - start_timestamp: expect.any(Number), - timestamp: expect.any(Number), - trace_id: expect.any(String), - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates ai related spans with sendDefaultPii: false', async () => { - await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan] = container.items; + + // [0] First generateText — invoke_agent (no explicit telemetry, no PII) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(30); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + + // [1] First generateText — generate_content (doGenerate, no PII) + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('mock-provider'); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + + // [2] Second generateText — invoke_agent (explicit telemetry enabled) + expect(thirdSpan!.name).toBe('invoke_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"Where is the second span?"}]', + ); + expect(thirdSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + ); + + // [3] Second generateText — generate_content (doGenerate with telemetry) + expect(fourthSpan!.name).toBe('generate_content mock-model-id'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(fourthSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toContain('Second span here!'); + + // [4] Third generateText — invoke_agent (tool call) + expect(fifthSpan!.name).toBe('invoke_agent'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(fifthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(fifthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(25); + expect(fifthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(40); + + // [5] Third generateText — generate_content (doGenerate with tools) + expect(sixthSpan!.name).toBe('generate_content mock-model-id'); + expect(sixthSpan!.status).toBe('ok'); + expect(sixthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(sixthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); + + // [6] Tool execution + expect(seventhSpan!.name).toBe('execute_tool getWeather'); + expect(seventhSpan!.status).toBe('ok'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); + }, + }) + .start() + .completed(); }); }); createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates ai related spans with sendDefaultPii: true', async () => { - await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan] = container.items; + + // [0] First generateText — invoke_agent (PII auto-enabled) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"Where is the first span?"}]', + ); + expect(firstSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ); + + // [1] First doGenerate with PII + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toContain('First span here!'); + + // [2] Second generateText — invoke_agent (explicit telemetry) + expect(thirdSpan!.name).toBe('invoke_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"Where is the second span?"}]', + ); + + // [3] Second doGenerate + expect(fourthSpan!.name).toBe('generate_content mock-model-id'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [4] Third generateText — invoke_agent (tool call prompt) + expect(fifthSpan!.name).toBe('invoke_agent'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(fifthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + ); + expect(fifthSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + + // [5] Third doGenerate with available tools + expect(sixthSpan!.name).toBe('generate_content mock-model-id'); + expect(sixthSpan!.status).toBe('ok'); + expect(sixthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toContain('getWeather'); + expect(sixthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); + + // [6] Tool execution with PII + expect(seventhSpan!.name).toBe('execute_tool getWeather'); + expect(seventhSpan!.status).toBe('ok'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE].value).toBe( + 'Get the current weather for a location', + ); + expect(seventhSpan!.attributes[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeDefined(); + expect(seventhSpan!.attributes[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeDefined(); + }, + }) + .start() + .completed(); }); }); createEsmAndCjsTests(__dirname, 'scenario-error-in-tool.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - const expectedTransaction = { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.settings.maxSteps': 1, - 'vercel.ai.streaming': false, - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'internal_error', - }), - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - expect.objectContaining({ - data: { - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.operationId': 'ai.toolCall', - }, - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'internal_error', - }), - ]), - - tags: { - 'test-tag': 'test-value', - }, - }; - let traceId: string = 'unset-trace-id'; let spanId: string = 'unset-span-id'; - const expectedError = { - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - }, - }, - exception: { - values: expect.arrayContaining([ - expect.objectContaining({ - type: 'AI_ToolExecutionError', - value: 'Error executing tool getWeather: Error in tool', - }), - ]), - }, - tags: { - 'test-tag': 'test-value', - }, - }; - await createRunner() .expect({ transaction: transaction => { - expect(transaction).toMatchObject(expectedTransaction); + expect(transaction.transaction).toBe('main'); + // gen_ai spans should be empty in transaction + expect(transaction.spans).toEqual([]); traceId = transaction.contexts!.trace!.trace_id; spanId = transaction.contexts!.trace!.span_id; }, }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] invoke_agent (errored due to tool error) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.status).toBe('error'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + + // [1] generate_content (doGenerate, succeeded) + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); + + // [2] execute_tool (errored) + expect(thirdSpan!.name).toBe('execute_tool getWeather'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(thirdSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + }, + }) .expect({ event: event => { - expect(event).toMatchObject(expectedError); + expect(event.exception?.values).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'AI_ToolExecutionError', + value: 'Error executing tool getWeather: Error in tool', + }), + ]), + ); + expect(event.tags).toMatchObject({ 'test-tag': 'test-value' }); expect(event.contexts!.trace!.trace_id).toBe(traceId); expect(event.contexts!.trace!.span_id).toBe(spanId); }, @@ -612,101 +240,6 @@ describe('Vercel AI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error in tool in express server', async () => { - const expectedTransaction = { - transaction: 'GET /test/error-in-tool', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.settings.maxSteps': 1, - 'vercel.ai.streaming': false, - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'internal_error', - }), - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - expect.objectContaining({ - data: { - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.operationId': 'ai.toolCall', - }, - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'internal_error', - }), - ]), - - tags: { - 'test-tag': 'test-value', - }, - }; - - const expectedError = { - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - }, - }, - exception: { - values: expect.arrayContaining([ - expect.objectContaining({ - type: 'AI_ToolExecutionError', - value: 'Error executing tool getWeather: Error in tool', - }), - ]), - }, - tags: { - 'test-tag': 'test-value', - }, - }; - let transactionEvent: Event | undefined; let errorEvent: Event | undefined; @@ -716,6 +249,29 @@ describe('Vercel AI integration', () => { transactionEvent = transaction; }, }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] invoke_agent (errored) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.status).toBe('error'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + + // [1] generate_content (doGenerate, succeeded) + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [2] execute_tool (errored) + expect(thirdSpan!.name).toBe('execute_tool getWeather'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(thirdSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + }, + }) .expect({ event: event => { errorEvent = event; @@ -727,11 +283,19 @@ describe('Vercel AI integration', () => { await runner.completed(); expect(transactionEvent).toBeDefined(); - expect(errorEvent).toBeDefined(); - - expect(transactionEvent).toMatchObject(expectedTransaction); + expect(transactionEvent!.transaction).toBe('GET /test/error-in-tool'); + expect(transactionEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); - expect(errorEvent).toMatchObject(expectedError); + expect(errorEvent).toBeDefined(); + expect(errorEvent!.exception?.values).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'AI_ToolExecutionError', + value: 'Error executing tool getWeather: Error in tool', + }), + ]), + ); + expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); expect(errorEvent!.contexts!.trace!.span_id).toBe(transactionEvent!.contexts!.trace!.span_id); }); @@ -739,37 +303,30 @@ describe('Vercel AI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-late-model-id.mjs', 'instrument.mjs', (createRunner, test) => { test('sets op correctly even when model ID is not available at span start', async () => { - const expectedTransaction = { - transaction: 'main', - spans: expect.arrayContaining([ - // The generateText span should have the correct op even though model ID was not available at span start - expect.objectContaining({ - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - data: expect.objectContaining({ - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - }), - }), - // The doGenerate span - name stays as 'generateText.doGenerate' since model ID is missing - expect.objectContaining({ - description: 'generateText.doGenerate', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - data: expect.objectContaining({ - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - }), - }), - ]), - }; - - await createRunner().expect({ transaction: expectedTransaction }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] invoke_agent + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.vercelai.otel'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('invoke_agent'); + + // [1] generate_content (doGenerate) + expect(secondSpan!.name).toBe('generateText.doGenerate'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.vercelai.otel'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('generate_content'); + }, + }) + .start() + .completed(); }); }); @@ -781,18 +338,22 @@ describe('Vercel AI integration', () => { test('extracts system instructions from messages', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([ - { type: 'text', content: 'You are a helpful assistant' }, - ]), - }), - }), - ]), + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] invoke_agent (carries system instructions) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE].value).toBe( + JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), + ); + + // [1] generate_content + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); }, }) .start() @@ -809,27 +370,27 @@ describe('Vercel AI integration', () => { test('truncates messages when they exceed byte limit', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: { - transaction: 'main', - spans: expect.arrayContaining([ - // First call: Last message truncated (only C's remain, D's are cropped) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^\[.*"(?:text|content)":"C+".*\]$/), - }), - }), - // Second call: Last message is small and kept intact - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringContaining( - 'This is a small message that fits within the limit', - ), - }), - }), - ]), + span: container => { + expect(container.items).toHaveLength(4); + const [firstSpan, , thirdSpan] = container.items; + + // [0] First call — invoke_agent: last message truncated (only C's remain, D's cropped) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[.*"(?:text|content)":"C+".*\]$/, + ); + + // [2] Second call — invoke_agent: last message is small and kept intact + expect(thirdSpan!.name).toBe('invoke_agent'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain( + 'This is a small message that fits within the limit', + ); }, }) .start() @@ -840,110 +401,79 @@ describe('Vercel AI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates embedding related spans with sendDefaultPii: false', async () => { - const expectedTransaction = { - transaction: 'main', - spans: expect.arrayContaining([ - // embed doEmbed span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 10, - }), - description: 'embeddings mock-model-id', - op: 'gen_ai.embeddings', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // embedMany doEmbed span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }), - description: 'embeddings mock-model-id', - op: 'gen_ai.embeddings', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - ]), - }; - - await createRunner().expect({ transaction: expectedTransaction }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] embed doEmbed + expect(firstSpan!.name).toBe('embeddings mock-model-id'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + + // [1] embedMany doEmbed + expect(secondSpan!.name).toBe('embeddings mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(20); + }, + }) + .start() + .completed(); }); }); createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates embedding related spans with sendDefaultPii: true', async () => { - const expectedTransaction = { - transaction: 'main', - spans: expect.arrayContaining([ - // embed doEmbed span with input - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: 'Embedding test!', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 10, - }), - description: 'embeddings mock-model-id', - op: 'gen_ai.embeddings', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // embedMany doEmbed span with input - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: '["First input","Second input"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }), - description: 'embeddings mock-model-id', - op: 'gen_ai.embeddings', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - ]), - }; - - await createRunner().expect({ transaction: expectedTransaction }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] embed doEmbed with input + expect(firstSpan!.name).toBe('embeddings mock-model-id'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(firstSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE].value).toBe('Embedding test!'); + + // [1] embedMany doEmbed with input + expect(secondSpan!.name).toBe('embeddings mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(secondSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE].value).toBe( + '["First input","Second input"]', + ); + }, + }) + .start() + .completed(); }); }); createEsmAndCjsTests(__dirname, 'scenario-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('does not overwrite conversation id set via Sentry.setConversationId with responseId from provider metadata', async () => { await createRunner() + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - op: 'gen_ai.invoke_agent', - data: expect.objectContaining({ - 'gen_ai.conversation.id': 'conv-a', - }), - }), - expect.objectContaining({ - op: 'gen_ai.generate_content', - data: expect.objectContaining({ - 'gen_ai.conversation.id': 'conv-a', - }), - }), - ]), + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] invoke_agent with user-set conversation id + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['gen_ai.conversation.id'].value).toBe('conv-a'); + + // [1] generate_content also inherits the conversation id + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes['gen_ai.conversation.id'].value).toBe('conv-a'); }, }) .start() @@ -960,22 +490,27 @@ describe('Vercel AI integration', () => { (createRunner, test) => { test('does not truncate input messages when enableTruncation is false', async () => { await createRunner() + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: { - transaction: 'main', - spans: expect.arrayContaining([ - // Multiple messages should all be preserved (no popping to last message only) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { role: 'user', content: longContent }, - { role: 'assistant', content: 'Some reply' }, - { role: 'user', content: 'Follow-up question' }, - ]), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - }), - }), - ]), + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] invoke_agent — input messages preserved in full (no truncation) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + JSON.stringify([ + { role: 'user', content: longContent }, + { role: 'assistant', content: 'Some reply' }, + { role: 'user', content: 'Follow-up question' }, + ]), + ); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + + // [1] generate_content + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); }, }) .start() @@ -991,12 +526,21 @@ describe('Vercel AI integration', () => { await createRunner() .expect({ span: container => { - const spans = container.items; + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), - ); - expect(chatSpan).toBeDefined(); + // [0] generate_content — in streaming mode, doGenerate ends first + expect(firstSpan!.name).toBe('generate_content mock-model-id'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [1] invoke_agent — carries the full (untruncated) input messages + expect(secondSpan!.name).toBe('invoke_agent'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain(streamingLongContent); + + // [2] main — root span (streamed alongside) + expect(thirdSpan!.name).toBe('main'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('function'); }, }) .start() @@ -1013,16 +557,26 @@ describe('Vercel AI integration', () => { await createRunner() .expect({ span: container => { - const spans = container.items; - - // With explicit enableTruncation: true, content should be truncated despite streaming. - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] generate_content — in streaming mode, doGenerate ends first + expect(firstSpan!.name).toBe('generate_content mock-model-id'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [1] invoke_agent — content truncated despite streaming (explicit enableTruncation: true) + expect(secondSpan!.name).toBe('invoke_agent'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"AAAA/, ); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( streamingLongContent.length, ); + + // [2] main — root span + expect(thirdSpan!.name).toBe('main'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('function'); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index e59a5545d7cf..3fc6dd19f1ba 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -1,15 +1,10 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_INPUT_MESSAGES_ATTRIBUTE, - GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, - GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE, - GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, - GEN_AI_RESPONSE_ID_ATTRIBUTE, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, GEN_AI_SYSTEM_ATTRIBUTE, GEN_AI_TOOL_CALL_ID_ATTRIBUTE, @@ -29,415 +24,75 @@ describe('Vercel AI integration (V5)', () => { cleanupChildProcesses(); }); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - no telemetry config, should enable telemetry but not record inputs/outputs when sendDefaultPii: false - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Second span - explicitly enabled telemetry but recordInputs/recordOutputs not set, should not record when sendDefaultPii: false - expect.objectContaining({ - data: { - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.model.provider': 'mock-provider', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.settings.maxRetries': 2, - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.streaming': false, - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.timestamp': expect.any(String), - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Third span - explicit telemetry enabled, should record inputs/outputs regardless of sendDefaultPii - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', - 'vercel.ai.response.finishReason': 'stop', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Fourth span - doGenerate for explicit telemetry enabled call - expect.objectContaining({ - data: { - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.model.provider': 'mock-provider', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.settings.maxRetries': 2, - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.streaming': false, - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.id': expect.any(String), - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - 'vercel.ai.response.timestamp': expect.any(String), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Fifth span - tool call generateText span - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Sixth span - tool call doGenerate span - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Seventh span - tool call execution span - // Note: gen_ai.tool.description is NOT present when sendDefaultPii: false because ai.prompt.tools is not recorded - expect.objectContaining({ - data: { - 'vercel.ai.operationId': 'ai.toolCall', - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - ]), - }; - - const EXPECTED_AVAILABLE_TOOLS_JSON = - '[{"type":"function","name":"getWeather","description":"Get the current weather for a location","inputSchema":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"location":{"type":"string"}},"required":["location"],"additionalProperties":false}}]'; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - no telemetry config, should enable telemetry AND record inputs/outputs when sendDefaultPii: true - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"Where is the first span?"}]', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the first span?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Second span - doGenerate for first call, should also include input/output fields when sendDefaultPii: true - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"user","content":[{"type":"text","text":"Where is the first span?"}]}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Third span - explicitly enabled telemetry, should record inputs/outputs regardless of sendDefaultPii - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Fourth span - doGenerate for explicitly enabled telemetry call - expect.objectContaining({ - data: { - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.model.provider': 'mock-provider', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.settings.maxRetries': 2, - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.streaming': false, - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.id': expect.any(String), - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - 'vercel.ai.response.timestamp': expect.any(String), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Fifth span - tool call generateText span (should include prompts when sendDefaultPii: true) - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"What is the weather in San Francisco?"}]', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather in San Francisco?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Sixth span - tool call doGenerate span (should include prompts when sendDefaultPii: true) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', - 'vercel.ai.prompt.toolChoice': expect.any(String), - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: EXPECTED_AVAILABLE_TOOLS_JSON, - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Seventh span - tool call execution span - expect.objectContaining({ - data: expect.objectContaining({ - 'vercel.ai.operationId': 'ai.toolCall', - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE]: 'Get the current weather for a location', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_INPUT_ATTRIBUTE]: expect.any(String), - [GEN_AI_TOOL_OUTPUT_ATTRIBUTE]: expect.any(String), - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates ai related spans with sendDefaultPii: false', async () => { - await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan] = container.items; + + // [0] First generateText — invoke_agent (no explicit telemetry, no PII) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(30); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + + // [1] First generateText — generate_content (doGenerate, no PII) + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('mock-provider'); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + + // [2] Second generateText — invoke_agent (explicit telemetry enabled) + expect(thirdSpan!.name).toBe('invoke_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"Where is the second span?"}]', + ); + expect(thirdSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + ); + + // [3] Second generateText — generate_content (doGenerate with PII) + expect(fourthSpan!.name).toBe('generate_content mock-model-id'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [4] Third generateText — invoke_agent (with tool call, no PII) + expect(fifthSpan!.name).toBe('invoke_agent'); + expect(fifthSpan!.status).toBe('ok'); + + // [5] Third generateText — generate_content (doGenerate) + expect(sixthSpan!.name).toBe('generate_content mock-model-id'); + expect(sixthSpan!.status).toBe('ok'); + + // [6] Tool execution + expect(seventhSpan!.name).toBe('execute_tool getWeather'); + expect(seventhSpan!.status).toBe('ok'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); + }, + }) + .start() + .completed(); }); }, { @@ -453,7 +108,74 @@ describe('Vercel AI integration (V5)', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('creates ai related spans with sendDefaultPii: true', async () => { - await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan] = container.items; + + // [0] First generateText — invoke_agent (PII auto-enabled) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"Where is the first span?"}]', + ); + expect(firstSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ); + + // [1] First doGenerate with PII + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toContain('First span here!'); + + // [2] Second generateText — invoke_agent (explicit telemetry) + expect(thirdSpan!.name).toBe('invoke_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"Where is the second span?"}]', + ); + + // [3] Second doGenerate + expect(fourthSpan!.name).toBe('generate_content mock-model-id'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [4] Third generateText — invoke_agent (tool call prompt) + expect(fifthSpan!.name).toBe('invoke_agent'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + ); + + // [5] Third doGenerate with available tools + expect(sixthSpan!.name).toBe('generate_content mock-model-id'); + expect(sixthSpan!.status).toBe('ok'); + expect(sixthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toBeDefined(); + expect(sixthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); + + // [6] Tool execution with PII + expect(seventhSpan!.name).toBe('execute_tool getWeather'); + expect(seventhSpan!.status).toBe('ok'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE].value).toBe( + 'Get the current weather for a location', + ); + expect(seventhSpan!.attributes[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeDefined(); + expect(seventhSpan!.attributes[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeDefined(); + }, + }) + .start() + .completed(); }); }, { @@ -469,84 +191,6 @@ describe('Vercel AI integration (V5)', () => { 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - const expectedTransaction = { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.response.finishReason': 'tool-calls', - }, - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - }), - expect.objectContaining({ - data: { - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - expect.objectContaining({ - data: { - 'vercel.ai.operationId': 'ai.toolCall', - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }, - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'internal_error', - }), - ]), - }; - - const expectedError = { - level: 'error', - tags: expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - }; - let transactionEvent: Event | undefined; let errorEvent: Event | undefined; @@ -556,6 +200,27 @@ describe('Vercel AI integration (V5)', () => { transactionEvent = transaction; }, }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] invoke_agent + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + + // [1] generate_content (doGenerate) + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [2] execute_tool (errored) + expect(thirdSpan!.name).toBe('execute_tool getWeather'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(thirdSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + }, + }) .expect({ event: event => { errorEvent = event; @@ -565,10 +230,16 @@ describe('Vercel AI integration (V5)', () => { .completed(); expect(transactionEvent).toBeDefined(); - expect(transactionEvent).toMatchObject(expectedTransaction); + expect(transactionEvent!.transaction).toBe('main'); expect(errorEvent).toBeDefined(); - expect(errorEvent).toMatchObject(expectedError); + expect(errorEvent!.level).toBe('error'); + expect(errorEvent!.tags).toEqual( + expect.objectContaining({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }), + ); // Trace id should be the same for the transaction and error event expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); @@ -587,7 +258,27 @@ describe('Vercel AI integration (V5)', () => { 'instrument.mjs', (createRunner, test) => { test('creates ai related spans with v5', async () => { - await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const [firstSpan, secondSpan, , , fifthSpan, sixthSpan, seventhSpan] = container.items; + + // invoke_agent spans at [0], [2], [4] + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + + // generate_content spans at [1], [3], [5] + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(sixthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // execute_tool at [6] + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + }, + }) + .start() + .completed(); }); }, { diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts index 1b030804f8d2..9c4e0ccd005e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts @@ -1,14 +1,11 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_INPUT_MESSAGES_ATTRIBUTE, - GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE, GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, - GEN_AI_RESPONSE_ID_ATTRIBUTE, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, GEN_AI_SYSTEM_ATTRIBUTE, GEN_AI_TOOL_CALL_ID_ATTRIBUTE, @@ -28,419 +25,75 @@ describe('Vercel AI integration (V6)', () => { cleanupChildProcesses(); }); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - no telemetry config, should enable telemetry but not record inputs/outputs when sendDefaultPii: false - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.request.headers.user-agent': expect.any(String), - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Second span - explicitly enabled telemetry but recordInputs/recordOutputs not set, should not record when sendDefaultPii: false - expect.objectContaining({ - data: expect.objectContaining({ - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.request.headers.user-agent': expect.any(String), - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.settings.maxRetries': 2, - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.streaming': false, - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.timestamp': expect.any(String), - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Third span - explicit telemetry enabled, should record inputs/outputs regardless of sendDefaultPii - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', - 'vercel.ai.request.headers.user-agent': expect.any(String), - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Fourth span - doGenerate for explicit telemetry enabled call - expect.objectContaining({ - data: expect.objectContaining({ - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.request.headers.user-agent': expect.any(String), - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.settings.maxRetries': 2, - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.streaming': false, - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.timestamp': expect.any(String), - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Fifth span - tool call generateText span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.request.headers.user-agent': expect.any(String), - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Sixth span - tool call doGenerate span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.request.headers.user-agent': expect.any(String), - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Seventh span - tool call execution span - // Note: gen_ai.tool.description is NOT present when sendDefaultPii: false because ai.prompt.tools is not recorded - expect.objectContaining({ - data: expect.objectContaining({ - 'vercel.ai.operationId': 'ai.toolCall', - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - ]), - }; - - const EXPECTED_AVAILABLE_TOOLS_JSON = - '[{"type":"function","name":"getWeather","description":"Get the current weather for a location","inputSchema":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"location":{"type":"string"}},"required":["location"],"additionalProperties":false}}]'; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - no telemetry config, should enable telemetry AND record inputs/outputs when sendDefaultPii: true - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"Where is the first span?"}]', - 'vercel.ai.request.headers.user-agent': expect.any(String), - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the first span?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Second span - doGenerate for first call, should also include input/output fields when sendDefaultPii: true - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.request.headers.user-agent': expect.any(String), - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"user","content":[{"type":"text","text":"Where is the first span?"}]}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Third span - explicitly enabled telemetry, should record inputs/outputs regardless of sendDefaultPii - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"Where is the second span?"}]', - 'vercel.ai.request.headers.user-agent': expect.any(String), - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Where is the second span?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Fourth span - doGenerate for explicitly enabled telemetry call - expect.objectContaining({ - data: expect.objectContaining({ - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.request.headers.user-agent': expect.any(String), - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.settings.maxRetries': 2, - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.streaming': false, - 'vercel.ai.response.finishReason': 'stop', - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.timestamp': expect.any(String), - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Fifth span - tool call generateText span (should include prompts when sendDefaultPii: true) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.prompt': '[{"role":"user","content":"What is the weather in San Francisco?"}]', - 'vercel.ai.request.headers.user-agent': expect.any(String), - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather in San Francisco?"}]', - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Sixth span - tool call doGenerate span (should include prompts when sendDefaultPii: true) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.request.headers.user-agent': expect.any(String), - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), - [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: - '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', - 'vercel.ai.prompt.toolChoice': expect.any(String), - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: EXPECTED_AVAILABLE_TOOLS_JSON, - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Seventh span - tool call execution span - expect.objectContaining({ - data: expect.objectContaining({ - 'vercel.ai.operationId': 'ai.toolCall', - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE]: 'Get the current weather for a location', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_INPUT_ATTRIBUTE]: expect.any(String), - [GEN_AI_TOOL_OUTPUT_ATTRIBUTE]: expect.any(String), - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates ai related spans with sendDefaultPii: false', async () => { - await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan] = container.items; + + // [0] First generateText — invoke_agent (no explicit telemetry, no PII) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(30); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + + // [1] First generateText — generate_content (doGenerate, no PII) + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('mock-provider'); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + + // [2] Second generateText — invoke_agent (explicit telemetry enabled) + expect(thirdSpan!.name).toBe('invoke_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"Where is the second span?"}]', + ); + expect(thirdSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + ); + + // [3] Second generateText — generate_content (doGenerate with PII) + expect(fourthSpan!.name).toBe('generate_content mock-model-id'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [4] Third generateText — invoke_agent (with tool call, no PII) + expect(fifthSpan!.name).toBe('invoke_agent'); + expect(fifthSpan!.status).toBe('ok'); + + // [5] Third generateText — generate_content (doGenerate) + expect(sixthSpan!.name).toBe('generate_content mock-model-id'); + expect(sixthSpan!.status).toBe('ok'); + + // [6] Tool execution + expect(seventhSpan!.name).toBe('execute_tool getWeather'); + expect(seventhSpan!.status).toBe('ok'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); + }, + }) + .start() + .completed(); }); }, { @@ -456,7 +109,71 @@ describe('Vercel AI integration (V6)', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('creates ai related spans with sendDefaultPii: true', async () => { - await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan] = container.items; + + // [0] First generateText — invoke_agent (PII auto-enabled) + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"Where is the first span?"}]', + ); + expect(firstSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ); + + // [1] First doGenerate with PII + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toContain('First span here!'); + + // [2] Second generateText — invoke_agent (explicit telemetry) + expect(thirdSpan!.name).toBe('invoke_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + + // [3] Second doGenerate + expect(fourthSpan!.name).toBe('generate_content mock-model-id'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [4] Third generateText — invoke_agent (tool call prompt) + expect(fifthSpan!.name).toBe('invoke_agent'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + ); + + // [5] Third doGenerate with available tools + expect(sixthSpan!.name).toBe('generate_content mock-model-id'); + expect(sixthSpan!.status).toBe('ok'); + expect(sixthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toBeDefined(); + expect(sixthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); + + // [6] Tool execution with PII + expect(seventhSpan!.name).toBe('execute_tool getWeather'); + expect(seventhSpan!.status).toBe('ok'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + expect(seventhSpan!.attributes[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE].value).toBe( + 'Get the current weather for a location', + ); + expect(seventhSpan!.attributes[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeDefined(); + expect(seventhSpan!.attributes[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeDefined(); + }, + }) + .start() + .completed(); }); }, { @@ -472,86 +189,6 @@ describe('Vercel AI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - const expectedTransaction = { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText', - 'vercel.ai.pipeline.name': 'generateText', - 'vercel.ai.request.headers.user-agent': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - 'vercel.ai.response.finishReason': 'tool-calls', - }), - description: 'invoke_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - }), - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - 'vercel.ai.model.provider': 'mock-provider', - 'vercel.ai.operationId': 'ai.generateText.doGenerate', - 'vercel.ai.pipeline.name': 'generateText.doGenerate', - 'vercel.ai.request.headers.user-agent': expect.any(String), - 'vercel.ai.response.finishReason': 'tool-calls', - 'vercel.ai.response.id': expect.any(String), - 'vercel.ai.response.model': 'mock-model-id', - 'vercel.ai.response.timestamp': expect.any(String), - 'vercel.ai.settings.maxRetries': 2, - 'vercel.ai.streaming': false, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model-id', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'mock-provider', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - expect.objectContaining({ - data: expect.objectContaining({ - 'vercel.ai.operationId': 'ai.toolCall', - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'internal_error', - }), - ]), - }; - - const expectedError = { - level: 'error', - tags: expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - }; - let transactionEvent: Event | undefined; let errorEvent: Event | undefined; @@ -561,6 +198,27 @@ describe('Vercel AI integration (V6)', () => { transactionEvent = transaction; }, }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] invoke_agent + expect(firstSpan!.name).toBe('invoke_agent'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + + // [1] generate_content (doGenerate) + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [2] execute_tool (errored) + expect(thirdSpan!.name).toBe('execute_tool getWeather'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(thirdSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + }, + }) .expect({ event: event => { errorEvent = event; @@ -570,10 +228,16 @@ describe('Vercel AI integration (V6)', () => { .completed(); expect(transactionEvent).toBeDefined(); - expect(transactionEvent).toMatchObject(expectedTransaction); + expect(transactionEvent!.transaction).toBe('main'); expect(errorEvent).toBeDefined(); - expect(errorEvent).toMatchObject(expectedError); + expect(errorEvent!.level).toBe('error'); + expect(errorEvent!.tags).toEqual( + expect.objectContaining({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }), + ); // Trace id should be the same for the transaction and error event expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); @@ -592,7 +256,27 @@ describe('Vercel AI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('creates ai related spans with v6', async () => { - await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const [firstSpan, secondSpan, , , fifthSpan, sixthSpan, seventhSpan] = container.items; + + // invoke_agent spans at [0], [2], [4] + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + + // generate_content spans at [1], [3], [5] + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(sixthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // execute_tool at [6] + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + }, + }) + .start() + .completed(); }); }, { @@ -608,71 +292,46 @@ describe('Vercel AI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('creates spans for ToolLoopAgent with tool calls', async () => { - const expectedTransaction = { - transaction: 'main', - spans: expect.arrayContaining([ - // ToolLoopAgent outer span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', - }), - description: 'invoke_agent weather_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // First doGenerate span (returns tool-calls) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Tool execution span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', - [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - }), - description: 'execute_tool getWeather', - op: 'gen_ai.execute_tool', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - // Second doGenerate span (returns final text) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - }), - description: 'generate_content mock-model-id', - op: 'gen_ai.generate_content', - origin: 'auto.vercelai.otel', - status: 'ok', - }), - ]), - }; - - await createRunner().expect({ transaction: expectedTransaction }).start().completed(); + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] invoke_agent (ToolLoopAgent outer span) + expect(firstSpan!.name).toBe('invoke_agent weather_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + + // [1] First doGenerate (returns tool-calls) + expect(secondSpan!.name).toBe('generate_content mock-model-id'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE].value).toBe('["tool-calls"]'); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); + + // [2] Tool execution + expect(thirdSpan!.name).toBe('execute_tool getWeather'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(thirdSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + expect(thirdSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); + expect(thirdSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); + + // [3] Second doGenerate (returns final text) + expect(fourthSpan!.name).toBe('generate_content mock-model-id'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE].value).toBe('["stop"]'); + expect(fourthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(fourthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(25); + }, + }) + .start() + .completed(); }); }, { From d51f4f26758ad952d9a5e7fd3afa68cda23c5cb8 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 07/18] Update Vercel AI E2E tests Co-Authored-By: Claude Opus 4.6 --- .../nextjs-15/tests/ai-error.test.ts | 16 +++--- .../nextjs-15/tests/ai-test.test.ts | 52 ++++++++++--------- .../nextjs-16/tests/ai-error.test.ts | 16 +++--- .../nextjs-16/tests/ai-test.test.ts | 52 ++++++++++--------- 4 files changed, 76 insertions(+), 60 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/nextjs-15/tests/ai-error.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-15/tests/ai-error.test.ts index a8c39ec032ec..81bf9d04ba97 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-15/tests/ai-error.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-15/tests/ai-error.test.ts @@ -1,11 +1,16 @@ import { expect, test } from '@playwright/test'; -import { waitForTransaction, waitForError } from '@sentry-internal/test-utils'; +import { getSpanOp, waitForError, waitForStreamedSpans, waitForTransaction } from '@sentry-internal/test-utils'; test('should create AI spans with correct attributes and error linking', async ({ page }) => { const aiTransactionPromise = waitForTransaction('nextjs-15', async transactionEvent => { return transactionEvent.transaction === 'GET /ai-error-test'; }); + // gen_ai spans are extracted into a separate span v2 envelope item + const genAiSpansPromise = waitForStreamedSpans('nextjs-15', spans => + spans.some(span => getSpanOp(span) === 'gen_ai.invoke_agent'), + ); + const errorEventPromise = waitForError('nextjs-15', async errorEvent => { return errorEvent.exception?.values?.[0]?.value?.includes('Tool call failed'); }); @@ -13,21 +18,20 @@ test('should create AI spans with correct attributes and error linking', async ( await page.goto('/ai-error-test'); const aiTransaction = await aiTransactionPromise; + const genAiSpans = await genAiSpansPromise; const errorEvent = await errorEventPromise; expect(aiTransaction).toBeDefined(); expect(aiTransaction.transaction).toBe('GET /ai-error-test'); - const spans = aiTransaction.spans || []; - // Each generateText call should create 2 spans: one for the pipeline and one for doGenerate // Plus a span for the tool call // TODO: For now, this is sadly not fully working - the monkey patching of the ai package is not working // because of this, only spans that are manually opted-in at call time will be captured // this may be fixed by https://github.com/vercel/ai/pull/6716 in the future - const aiPipelineSpans = spans.filter(span => span.op === 'gen_ai.invoke_agent'); - const aiGenerateSpans = spans.filter(span => span.op === 'gen_ai.generate_content'); - const toolCallSpans = spans.filter(span => span.op === 'gen_ai.execute_tool'); + const aiPipelineSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + const aiGenerateSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.generate_content'); + const toolCallSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.execute_tool'); expect(aiPipelineSpans.length).toBeGreaterThanOrEqual(1); expect(aiGenerateSpans.length).toBeGreaterThanOrEqual(1); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-15/tests/ai-test.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-15/tests/ai-test.test.ts index 42c21e4f8c80..b76ea6eb9ff9 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-15/tests/ai-test.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-15/tests/ai-test.test.ts @@ -1,29 +1,33 @@ import { expect, test } from '@playwright/test'; -import { waitForTransaction } from '@sentry-internal/test-utils'; +import { getSpanOp, waitForStreamedSpans, waitForTransaction } from '@sentry-internal/test-utils'; test('should create AI spans with correct attributes', async ({ page }) => { const aiTransactionPromise = waitForTransaction('nextjs-15', async transactionEvent => { return transactionEvent.transaction === 'GET /ai-test'; }); + // gen_ai spans are extracted into a separate span v2 envelope item + const genAiSpansPromise = waitForStreamedSpans('nextjs-15', spans => + spans.some(span => getSpanOp(span) === 'gen_ai.invoke_agent'), + ); + await page.goto('/ai-test'); const aiTransaction = await aiTransactionPromise; + const genAiSpans = await genAiSpansPromise; expect(aiTransaction).toBeDefined(); expect(aiTransaction.transaction).toBe('GET /ai-test'); - const spans = aiTransaction.spans || []; - // We expect spans for the first 3 AI calls (4th is disabled) // Each generateText call should create 2 spans: one for the pipeline and one for doGenerate // Plus a span for the tool call // TODO: For now, this is sadly not fully working - the monkey patching of the ai package is not working // because of this, only spans that are manually opted-in at call time will be captured // this may be fixed by https://github.com/vercel/ai/pull/6716 in the future - const aiPipelineSpans = spans.filter(span => span.op === 'gen_ai.invoke_agent'); - const aiGenerateSpans = spans.filter(span => span.op === 'gen_ai.generate_content'); - const toolCallSpans = spans.filter(span => span.op === 'gen_ai.execute_tool'); + const aiPipelineSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + const aiGenerateSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.generate_content'); + const toolCallSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.execute_tool'); expect(aiPipelineSpans.length).toBeGreaterThanOrEqual(1); expect(aiGenerateSpans.length).toBeGreaterThanOrEqual(1); @@ -31,35 +35,35 @@ test('should create AI spans with correct attributes', async ({ page }) => { // First AI call - should have telemetry enabled and record inputs/outputs (sendDefaultPii: true) /* const firstPipelineSpan = aiPipelineSpans[0]; - expect(firstPipelineSpan?.data?.['vercel.ai.model.id']).toBe('mock-model-id'); - expect(firstPipelineSpan?.data?.['vercel.ai.model.provider']).toBe('mock-provider'); - expect(firstPipelineSpan?.data?.['vercel.ai.prompt']).toContain('Where is the first span?'); - expect(firstPipelineSpan?.data?.['gen_ai.output.messages']).toContain('First span here!'); - expect(firstPipelineSpan?.data?.['gen_ai.usage.input_tokens']).toBe(10); - expect(firstPipelineSpan?.data?.['gen_ai.usage.output_tokens']).toBe(20); */ + expect(firstPipelineSpan?.attributes?.['vercel.ai.model.id']?.value).toBe('mock-model-id'); + expect(firstPipelineSpan?.attributes?.['vercel.ai.model.provider']?.value).toBe('mock-provider'); + expect(firstPipelineSpan?.attributes?.['vercel.ai.prompt']?.value).toContain('Where is the first span?'); + expect(firstPipelineSpan?.attributes?.['gen_ai.output.messages']?.value).toContain('First span here!'); + expect(firstPipelineSpan?.attributes?.['gen_ai.usage.input_tokens']?.value).toBe(10); + expect(firstPipelineSpan?.attributes?.['gen_ai.usage.output_tokens']?.value).toBe(20); */ // Second AI call - explicitly enabled telemetry const secondPipelineSpan = aiPipelineSpans[0]; - expect(secondPipelineSpan?.data?.['vercel.ai.prompt']).toContain('Where is the second span?'); - expect(secondPipelineSpan?.data?.['gen_ai.output.messages']).toContain('Second span here!'); + expect(secondPipelineSpan?.attributes?.['vercel.ai.prompt']?.value).toContain('Where is the second span?'); + expect(secondPipelineSpan?.attributes?.['gen_ai.output.messages']?.value).toContain('Second span here!'); // Third AI call - with tool calls /* const thirdPipelineSpan = aiPipelineSpans[2]; - expect(thirdPipelineSpan?.data?.['vercel.ai.response.finishReason']).toBe('tool-calls'); - expect(thirdPipelineSpan?.data?.['gen_ai.usage.input_tokens']).toBe(15); - expect(thirdPipelineSpan?.data?.['gen_ai.usage.output_tokens']).toBe(25); */ + expect(thirdPipelineSpan?.attributes?.['vercel.ai.response.finishReason']?.value).toBe('tool-calls'); + expect(thirdPipelineSpan?.attributes?.['gen_ai.usage.input_tokens']?.value).toBe(15); + expect(thirdPipelineSpan?.attributes?.['gen_ai.usage.output_tokens']?.value).toBe(25); */ // Tool call span /* const toolSpan = toolCallSpans[0]; - expect(toolSpan?.data?.['vercel.ai.toolCall.name']).toBe('getWeather'); - expect(toolSpan?.data?.['vercel.ai.toolCall.id']).toBe('call-1'); - expect(toolSpan?.data?.['vercel.ai.toolCall.args']).toContain('San Francisco'); - expect(toolSpan?.data?.['vercel.ai.toolCall.result']).toContain('Sunny, 72°F'); */ + expect(toolSpan?.attributes?.['vercel.ai.toolCall.name']?.value).toBe('getWeather'); + expect(toolSpan?.attributes?.['vercel.ai.toolCall.id']?.value).toBe('call-1'); + expect(toolSpan?.attributes?.['vercel.ai.toolCall.args']?.value).toContain('San Francisco'); + expect(toolSpan?.attributes?.['vercel.ai.toolCall.result']?.value).toContain('Sunny, 72°F'); */ // Verify the fourth call was not captured (telemetry disabled) - const promptsInSpans = spans - .map(span => span.data?.['vercel.ai.prompt']) - .filter((prompt): prompt is string => prompt !== undefined); + const promptsInSpans = genAiSpans + .map(span => span.attributes?.['vercel.ai.prompt']?.value) + .filter((prompt): prompt is string => typeof prompt === 'string'); const hasDisabledPrompt = promptsInSpans.some(prompt => prompt.includes('Where is the third span?')); expect(hasDisabledPrompt).toBe(false); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/ai-error.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/ai-error.test.ts index 39e76bab0dde..62e6798773bd 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/ai-error.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/ai-error.test.ts @@ -1,11 +1,16 @@ import { expect, test } from '@playwright/test'; -import { waitForTransaction, waitForError } from '@sentry-internal/test-utils'; +import { getSpanOp, waitForError, waitForStreamedSpans, waitForTransaction } from '@sentry-internal/test-utils'; test('should create AI spans with correct attributes and error linking', async ({ page }) => { const aiTransactionPromise = waitForTransaction('nextjs-16', async transactionEvent => { return transactionEvent.transaction === 'GET /ai-error-test'; }); + // gen_ai spans are extracted into a separate span v2 envelope item + const genAiSpansPromise = waitForStreamedSpans('nextjs-16', spans => + spans.some(span => getSpanOp(span) === 'gen_ai.invoke_agent'), + ); + const errorEventPromise = waitForError('nextjs-16', async errorEvent => { return errorEvent.exception?.values?.[0]?.value?.includes('Tool call failed'); }); @@ -13,21 +18,20 @@ test('should create AI spans with correct attributes and error linking', async ( await page.goto('/ai-error-test'); const aiTransaction = await aiTransactionPromise; + const genAiSpans = await genAiSpansPromise; const errorEvent = await errorEventPromise; expect(aiTransaction).toBeDefined(); expect(aiTransaction.transaction).toBe('GET /ai-error-test'); - const spans = aiTransaction.spans || []; - // Each generateText call should create 2 spans: one for the pipeline and one for doGenerate // Plus a span for the tool call // TODO: For now, this is sadly not fully working - the monkey patching of the ai package is not working // because of this, only spans that are manually opted-in at call time will be captured // this may be fixed by https://github.com/vercel/ai/pull/6716 in the future - const aiPipelineSpans = spans.filter(span => span.op === 'gen_ai.invoke_agent'); - const aiGenerateSpans = spans.filter(span => span.op === 'gen_ai.generate_content'); - const toolCallSpans = spans.filter(span => span.op === 'gen_ai.execute_tool'); + const aiPipelineSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + const aiGenerateSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.generate_content'); + const toolCallSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.execute_tool'); expect(aiPipelineSpans.length).toBeGreaterThanOrEqual(1); expect(aiGenerateSpans.length).toBeGreaterThanOrEqual(1); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/ai-test.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/ai-test.test.ts index dcd129020035..89af644b1f21 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/ai-test.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/ai-test.test.ts @@ -1,29 +1,33 @@ import { expect, test } from '@playwright/test'; -import { waitForTransaction } from '@sentry-internal/test-utils'; +import { getSpanOp, waitForStreamedSpans, waitForTransaction } from '@sentry-internal/test-utils'; test('should create AI spans with correct attributes', async ({ page }) => { const aiTransactionPromise = waitForTransaction('nextjs-16', async transactionEvent => { return transactionEvent.transaction === 'GET /ai-test'; }); + // gen_ai spans are extracted into a separate span v2 envelope item + const genAiSpansPromise = waitForStreamedSpans('nextjs-16', spans => + spans.some(span => getSpanOp(span) === 'gen_ai.invoke_agent'), + ); + await page.goto('/ai-test'); const aiTransaction = await aiTransactionPromise; + const genAiSpans = await genAiSpansPromise; expect(aiTransaction).toBeDefined(); expect(aiTransaction.transaction).toBe('GET /ai-test'); - const spans = aiTransaction.spans || []; - // We expect spans for the first 3 AI calls (4th is disabled) // Each generateText call should create 2 spans: one for the pipeline and one for doGenerate // Plus a span for the tool call // TODO: For now, this is sadly not fully working - the monkey patching of the ai package is not working // because of this, only spans that are manually opted-in at call time will be captured // this may be fixed by https://github.com/vercel/ai/pull/6716 in the future - const aiPipelineSpans = spans.filter(span => span.op === 'gen_ai.invoke_agent'); - const aiGenerateSpans = spans.filter(span => span.op === 'gen_ai.generate_content'); - const toolCallSpans = spans.filter(span => span.op === 'gen_ai.execute_tool'); + const aiPipelineSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + const aiGenerateSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.generate_content'); + const toolCallSpans = genAiSpans.filter(span => getSpanOp(span) === 'gen_ai.execute_tool'); expect(aiPipelineSpans.length).toBeGreaterThanOrEqual(1); expect(aiGenerateSpans.length).toBeGreaterThanOrEqual(1); @@ -31,35 +35,35 @@ test('should create AI spans with correct attributes', async ({ page }) => { // First AI call - should have telemetry enabled and record inputs/outputs (sendDefaultPii: true) /* const firstPipelineSpan = aiPipelineSpans[0]; - expect(firstPipelineSpan?.data?.['vercel.ai.model.id']).toBe('mock-model-id'); - expect(firstPipelineSpan?.data?.['vercel.ai.model.provider']).toBe('mock-provider'); - expect(firstPipelineSpan?.data?.['vercel.ai.prompt']).toContain('Where is the first span?'); - expect(firstPipelineSpan?.data?.['gen_ai.output.messages']).toContain('First span here!'); - expect(firstPipelineSpan?.data?.['gen_ai.usage.input_tokens']).toBe(10); - expect(firstPipelineSpan?.data?.['gen_ai.usage.output_tokens']).toBe(20); */ + expect(firstPipelineSpan?.attributes?.['vercel.ai.model.id']?.value).toBe('mock-model-id'); + expect(firstPipelineSpan?.attributes?.['vercel.ai.model.provider']?.value).toBe('mock-provider'); + expect(firstPipelineSpan?.attributes?.['vercel.ai.prompt']?.value).toContain('Where is the first span?'); + expect(firstPipelineSpan?.attributes?.['gen_ai.output.messages']?.value).toContain('First span here!'); + expect(firstPipelineSpan?.attributes?.['gen_ai.usage.input_tokens']?.value).toBe(10); + expect(firstPipelineSpan?.attributes?.['gen_ai.usage.output_tokens']?.value).toBe(20); */ // Second AI call - explicitly enabled telemetry const secondPipelineSpan = aiPipelineSpans[0]; - expect(secondPipelineSpan?.data?.['vercel.ai.prompt']).toContain('Where is the second span?'); - expect(secondPipelineSpan?.data?.['gen_ai.output.messages']).toContain('Second span here!'); + expect(secondPipelineSpan?.attributes?.['vercel.ai.prompt']?.value).toContain('Where is the second span?'); + expect(secondPipelineSpan?.attributes?.['gen_ai.output.messages']?.value).toContain('Second span here!'); // Third AI call - with tool calls /* const thirdPipelineSpan = aiPipelineSpans[2]; - expect(thirdPipelineSpan?.data?.['vercel.ai.response.finishReason']).toBe('tool-calls'); - expect(thirdPipelineSpan?.data?.['gen_ai.usage.input_tokens']).toBe(15); - expect(thirdPipelineSpan?.data?.['gen_ai.usage.output_tokens']).toBe(25); */ + expect(thirdPipelineSpan?.attributes?.['vercel.ai.response.finishReason']?.value).toBe('tool-calls'); + expect(thirdPipelineSpan?.attributes?.['gen_ai.usage.input_tokens']?.value).toBe(15); + expect(thirdPipelineSpan?.attributes?.['gen_ai.usage.output_tokens']?.value).toBe(25); */ // Tool call span /* const toolSpan = toolCallSpans[0]; - expect(toolSpan?.data?.['vercel.ai.toolCall.name']).toBe('getWeather'); - expect(toolSpan?.data?.['vercel.ai.toolCall.id']).toBe('call-1'); - expect(toolSpan?.data?.['vercel.ai.toolCall.args']).toContain('San Francisco'); - expect(toolSpan?.data?.['vercel.ai.toolCall.result']).toContain('Sunny, 72°F'); */ + expect(toolSpan?.attributes?.['vercel.ai.toolCall.name']?.value).toBe('getWeather'); + expect(toolSpan?.attributes?.['vercel.ai.toolCall.id']?.value).toBe('call-1'); + expect(toolSpan?.attributes?.['vercel.ai.toolCall.args']?.value).toContain('San Francisco'); + expect(toolSpan?.attributes?.['vercel.ai.toolCall.result']?.value).toContain('Sunny, 72°F'); */ // Verify the fourth call was not captured (telemetry disabled) - const promptsInSpans = spans - .map(span => span.data?.['vercel.ai.prompt']) - .filter((prompt): prompt is string => prompt !== undefined); + const promptsInSpans = genAiSpans + .map(span => span.attributes?.['vercel.ai.prompt']?.value) + .filter((prompt): prompt is string => typeof prompt === 'string'); const hasDisabledPrompt = promptsInSpans.some(prompt => prompt.includes('Where is the third span?')); expect(hasDisabledPrompt).toBe(false); From bed75f70560b3cc4074469f5b6a293fe8931d07a Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 08/18] Update Anthropic Node tests Co-Authored-By: Claude Opus 4.6 --- .../suites/tracing/anthropic/test.ts | 1122 ++++++++--------- 1 file changed, 520 insertions(+), 602 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index e740c24071fd..61505134a7b0 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -1,4 +1,3 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_INPUT_MESSAGES_ATTRIBUTE, @@ -30,305 +29,14 @@ describe('Anthropic integration', () => { const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { transaction: 'main', - spans: expect.arrayContaining([ - // First span - basic message completion without PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'msg_mock123', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - // Second span - error handling - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }), - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'internal_error', - }), - // Third span - token counting (no response.text because recordOutputs=false by default) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - // Fourth span - models.retrieve - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'models', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.models', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - }), - description: 'models claude-3-haiku-20240307', - op: 'gen_ai.models', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - ]), }; const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { transaction: 'main', - spans: expect.arrayContaining([ - // First span - basic message completion with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the capital of France?"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'msg_mock123', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Hello from Anthropic mock!', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - expect.objectContaining({ - data: expect.objectContaining({ - 'http.request.method': 'POST', - 'http.request.method_original': 'POST', - 'http.response.status_code': 200, - 'otel.kind': 'CLIENT', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'http.client', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.http.otel.node_fetch', - 'url.path': '/anthropic/v1/messages', - 'url.query': '', - 'url.scheme': 'http', - }), - op: 'http.client', - origin: 'auto.http.otel.node_fetch', - status: 'ok', - }), - - // Second - error handling with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"This will fail"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - }), - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'internal_error', - }), - expect.objectContaining({ - data: expect.objectContaining({ - 'http.request.method': 'POST', - 'http.request.method_original': 'POST', - 'http.response.status_code': 404, - 'otel.kind': 'CLIENT', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'http.client', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.http.otel.node_fetch', - 'url.path': '/anthropic/v1/messages', - 'url.query': '', - 'url.scheme': 'http', - }), - op: 'http.client', - origin: 'auto.http.otel.node_fetch', - status: 'not_found', - }), - - // Third - token counting with PII (response.text is present because sendDefaultPii=true enables recordOutputs) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the capital of France?"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: '15', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - expect.objectContaining({ - data: expect.objectContaining({ - 'http.request.method': 'POST', - 'http.request.method_original': 'POST', - 'http.response.status_code': 200, - 'otel.kind': 'CLIENT', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'http.client', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.http.otel.node_fetch', - 'url.path': '/anthropic/v1/messages/count_tokens', - 'url.query': '', - 'url.scheme': 'http', - }), - op: 'http.client', - origin: 'auto.http.otel.node_fetch', - status: 'ok', - }), - - // Fourth - models.retrieve with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'models', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.models', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - }), - description: 'models claude-3-haiku-20240307', - op: 'gen_ai.models', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - expect.objectContaining({ - data: expect.objectContaining({ - 'http.request.method': 'GET', - 'http.request.method_original': 'GET', - 'http.response.status_code': 200, - 'otel.kind': 'CLIENT', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'http.client', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.http.otel.node_fetch', - 'url.path': '/anthropic/v1/models/claude-3-haiku-20240307', - 'url.query': '', - 'url.scheme': 'http', - 'user_agent.original': 'Anthropic/JS 0.63.0', - }), - op: 'http.client', - origin: 'auto.http.otel.node_fetch', - status: 'ok', - }), - - // Fifth - messages.create with stream: true - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the capital of France?"}]', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'msg_stream123', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Hello from stream!', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - expect.objectContaining({ - data: expect.objectContaining({ - 'http.request.method': 'POST', - 'http.request.method_original': 'POST', - 'http.response.status_code': 200, - 'otel.kind': 'CLIENT', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'http.client', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.http.otel.node_fetch', - 'url.path': '/anthropic/v1/messages', - 'url.query': '', - 'url.scheme': 'http', - 'user_agent.original': 'Anthropic/JS 0.63.0', - }), - op: 'http.client', - origin: 'auto.http.otel.node_fetch', - status: 'ok', - }), - - // Sixth - messages.stream - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - ]), }; const EXPECTED_TRANSACTION_WITH_OPTIONS = { transaction: 'main', - spans: expect.arrayContaining([ - // Check that custom options are respected - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response text when recordOutputs: true - }), - }), - // Check token counting with options - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: '15', // Present because recordOutputs=true is set in options - }), - op: 'gen_ai.chat', - }), - // Check models.retrieve with options - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'models', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - }), - op: 'gen_ai.models', - description: 'models claude-3-haiku-20240307', - }), - ]), }; const EXPECTED_MODEL_ERROR = { @@ -351,35 +59,92 @@ describe('Anthropic integration', () => { await createRunner() .ignore('event') .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .expect({ + span: container => { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] messages.create — basic message completion without PII + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_mock123'); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + + // [1] messages.create with error-model — error handling + expect(secondSpan!.name).toBe('chat error-model'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-model'); + + // [2] messages.countTokens — token counting + expect(thirdSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + + // [3] models.retrieve + expect(fourthSpan!.name).toBe('models claude-3-haiku-20240307'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('models'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.models'); + expect(fourthSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + }, + }) .start() .completed(); }); }); createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { - const chatSpan = (responseId: string) => - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: responseId, - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - status: 'ok', - }); - test('preserves .withResponse() and .asResponse() for non-streaming and streaming', async () => { await createRunner() .ignore('event') .expect({ transaction: { transaction: 'main', - spans: expect.arrayContaining([ - chatSpan('msg_withresponse'), - chatSpan('msg_withresponse'), - chatSpan('msg_stream_withresponse'), - ]), + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] .withResponse() — non-streaming + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_withresponse'); + + // [1] .asResponse() — non-streaming + expect(secondSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_withresponse'); + + // [2] streaming .withResponse() + expect(thirdSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_stream_withresponse'); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); }, }) .start() @@ -392,6 +157,37 @@ describe('Anthropic integration', () => { await createRunner() .expect({ event: EXPECTED_MODEL_ERROR }) .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .expect({ + span: container => { + expect(container.items).toHaveLength(5); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan] = container.items; + + // [0] messages.create — basic message completion + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_mock123'); + + // [1] messages.create with error-model — error handling + expect(secondSpan!.name).toBe('chat error-model'); + expect(secondSpan!.status).toBe('error'); + + // [2] messages.countTokens — token counting + expect(thirdSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + + // [3] models.retrieve + expect(fourthSpan!.name).toBe('models claude-3-haiku-20240307'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.models'); + + // [4] messages.create stream: true + messages.stream (both share this span due to pre-existing bug) + expect(fifthSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_stream123'); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + }, + }) .expect({ event: EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }) .start() .completed(); @@ -403,6 +199,66 @@ describe('Anthropic integration', () => { await createRunner() .expect({ event: EXPECTED_MODEL_ERROR }) .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }) + .expect({ + span: container => { + expect(container.items).toHaveLength(5); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan] = container.items; + + // [0] messages.create — basic message completion with PII + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"What is the capital of France?"}]', + ); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_mock123'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toBe('Hello from Anthropic mock!'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); + + // [1] messages.create with error-model — error handling with PII + expect(secondSpan!.name).toBe('chat error-model'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"user","content":"This will fail"}]', + ); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-model'); + + // [2] messages.countTokens — token counting with PII (response text records input_tokens as "15") + expect(thirdSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toBe('15'); + + // [3] models.retrieve with PII + expect(fourthSpan!.name).toBe('models claude-3-haiku-20240307'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.models'); + + // [4] messages.create stream: true + messages.stream (both share this span due to pre-existing bug) + // TODO: messages.stream() should produce its own distinct gen_ai span, but it + // currently does not (pre-existing bug). Once fixed, add an additional indexed span assertion. + expect(fifthSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(fifthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(fifthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_stream123'); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toBe('Hello from stream!'); + expect(fifthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(fifthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(fifthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + }, + }) .expect({ event: EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }) .start() .completed(); @@ -414,6 +270,48 @@ describe('Anthropic integration', () => { await createRunner() .expect({ event: EXPECTED_MODEL_ERROR }) .expect({ transaction: EXPECTED_TRANSACTION_WITH_OPTIONS }) + .expect({ + span: container => { + expect(container.items).toHaveLength(5); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan] = container.items; + + // [0] messages.create — chat span with custom PII options (input messages + response text recorded) + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_mock123'); + + // [1] messages.create with error-model — error handling + expect(secondSpan!.name).toBe('chat error-model'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + + // [2] messages.countTokens — token counting with options (response text = "15") + expect(thirdSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toBe('15'); + + // [3] models.retrieve with options + expect(fourthSpan!.name).toBe('models claude-3-haiku-20240307'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('models'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.models'); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + + // [4] messages.create stream: true + messages.stream (share this span due to pre-existing bug) + expect(fifthSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_stream123'); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + }, + }) .expect({ event: EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }) .start() .completed(); @@ -422,101 +320,98 @@ describe('Anthropic integration', () => { const EXPECTED_STREAM_SPANS_PII_FALSE = { transaction: 'main', - spans: expect.arrayContaining([ - // messages.create with stream: true - expect.objectContaining({ - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - data: expect.objectContaining({ - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'msg_stream_1', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["end_turn"]', - }), - }), - // messages.stream - expect.objectContaining({ - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - data: expect.objectContaining({ - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'msg_stream_1', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }), - }), - // messages.stream with redundant stream: true param - expect.objectContaining({ - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - data: expect.objectContaining({ - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'msg_stream_1', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }), - }), - ]), }; const EXPECTED_STREAM_SPANS_PII_TRUE = { transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - data: expect.objectContaining({ - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - // streamed text concatenated - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Hello from stream!', - }), - }), - expect.objectContaining({ - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - data: expect.objectContaining({ - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Hello from stream!', - }), - }), - expect.objectContaining({ - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - data: expect.objectContaining({ - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Hello from stream!', - }), - }), - ]), }; createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument.mjs', (createRunner, test) => { test('streams produce spans with token usage and metadata (PII false)', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_STREAM_SPANS_PII_FALSE }).start().completed(); + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_STREAM_SPANS_PII_FALSE }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] messages.create with stream: true + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_stream_1'); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE].value).toBe('["end_turn"]'); + + // [1] messages.stream (no request.stream attribute — distinguishes from the other two) + expect(secondSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_stream_1'); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + + // [2] messages.stream with redundant stream: true param + expect(thirdSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_stream_1'); + }, + }) + .start() + .completed(); }); }); createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('streams record response text when PII true', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_STREAM_SPANS_PII_TRUE }).start().completed(); + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_STREAM_SPANS_PII_TRUE }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] messages.create with stream: true — response text captured with PII + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(firstSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toBe('Hello from stream!'); + + // [1] messages.stream — response text captured with PII, no request.stream param + expect(secondSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(secondSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toBe('Hello from stream!'); + + // [2] messages.stream with redundant stream: true — response text captured with PII + expect(thirdSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toBe('Hello from stream!'); + }, + }) + .start() + .completed(); }); }); @@ -530,16 +425,19 @@ describe('Anthropic integration', () => { await createRunner() .ignore('event') .expect({ - transaction: { - spans: expect.arrayContaining([ - expect.objectContaining({ - op: 'gen_ai.chat', - data: expect.objectContaining({ - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: EXPECTED_TOOLS_JSON, - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: EXPECTED_TOOL_CALLS_JSON, - }), - }), - ]), + transaction: {}, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] messages.create with tools — available tools + tool calls recorded with PII + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toBe(EXPECTED_TOOLS_JSON); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE].value).toBe(EXPECTED_TOOL_CALLS_JSON); }, }) .start() @@ -557,17 +455,27 @@ describe('Anthropic integration', () => { await createRunner() .ignore('event') .expect({ - transaction: { - spans: expect.arrayContaining([ - expect.objectContaining({ - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - data: expect.objectContaining({ - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: EXPECTED_TOOLS_JSON, - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: EXPECTED_TOOL_CALLS_JSON, - }), - }), - ]), + transaction: {}, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] messages.create(stream: true) with tools — available tools + tool calls recorded with PII + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(firstSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toBe(EXPECTED_TOOLS_JSON); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE].value).toBe(EXPECTED_TOOL_CALLS_JSON); + + // [1] messages.stream — currently records as error since messages.stream doesn't get + // iterable semantics through the mock; this preserves observed behavior. + expect(secondSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); }, }) .start() @@ -578,97 +486,85 @@ describe('Anthropic integration', () => { // Additional error scenarios - Streaming errors const EXPECTED_STREAM_ERROR_SPANS = { transaction: 'main', - spans: expect.arrayContaining([ - // Error with messages.create on stream initialization - expect.objectContaining({ - description: 'chat error-stream-init', - op: 'gen_ai.chat', - status: 'internal_error', // Actual status coming from the instrumentation - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-stream-init', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - }), - }), - // Error with messages.stream on stream initialization - expect.objectContaining({ - description: 'chat error-stream-init', - op: 'gen_ai.chat', - status: 'internal_error', // Actual status coming from the instrumentation - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-stream-init', - }), - }), - // Error midway with messages.create on streaming - note: The stream is started successfully - // so we get a successful span with the content that was streamed before the error - expect.objectContaining({ - description: 'chat error-stream-midway', - op: 'gen_ai.chat', - status: 'ok', - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-stream-midway', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'This stream will ', // We received some data before error - }), - }), - // Error midway with messages.stream - same behavior, we get a span with the streamed data - expect.objectContaining({ - description: 'chat error-stream-midway', - op: 'gen_ai.chat', - status: 'ok', - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-stream-midway', - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'This stream will ', // We received some data before error - }), - }), - ]), }; createEsmAndCjsTests(__dirname, 'scenario-stream-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles streaming errors correctly', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_STREAM_ERROR_SPANS }).start().completed(); + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_STREAM_ERROR_SPANS }) + .expect({ + span: container => { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] messages.create(stream: true) error on stream init + expect(firstSpan!.name).toBe('chat error-stream-init'); + expect(firstSpan!.status).toBe('error'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-stream-init'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + + // [1] messages.stream error on stream init (no request.stream param) + expect(secondSpan!.name).toBe('chat error-stream-init'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-stream-init'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + + // [2] messages.create(stream: true) midway error — finishes 'ok' with partial text + expect(thirdSpan!.name).toBe('chat error-stream-midway'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-stream-midway'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toBe('This stream will '); + + // [3] messages.stream midway error — errors out + expect(fourthSpan!.name).toBe('chat error-stream-midway'); + expect(fourthSpan!.status).toBe('error'); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-stream-midway'); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + }, + }) + .start() + .completed(); }); }); // Additional error scenarios - Tool errors and model retrieval errors const EXPECTED_ERROR_SPANS = { transaction: 'main', - spans: expect.arrayContaining([ - // Invalid tool format error - expect.objectContaining({ - description: 'chat invalid-format', - op: 'gen_ai.chat', - status: 'internal_error', - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'invalid-format', - }), - }), - // Model retrieval error - expect.objectContaining({ - description: 'models nonexistent-model', - op: 'gen_ai.models', - status: 'internal_error', - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'nonexistent-model', - }), - }), - // Successful tool usage (for comparison) - expect.objectContaining({ - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - status: 'ok', - data: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: expect.stringContaining('tool_ok_1'), - }), - }), - ]), }; createEsmAndCjsTests(__dirname, 'scenario-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles tool errors and model retrieval errors correctly', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_ERROR_SPANS }).start().completed(); + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_ERROR_SPANS }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] messages.create with invalid-format — tool format error + expect(firstSpan!.name).toBe('chat invalid-format'); + expect(firstSpan!.status).toBe('error'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('invalid-format'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + + // [1] models.retrieve('nonexistent-model') — model retrieval error + expect(secondSpan!.name).toBe('models nonexistent-model'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('nonexistent-model'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.models'); + + // [2] Successful tool usage (for comparison) + expect(thirdSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE].value).toContain('tool_ok_1'); + }, + }) + .start() + .completed(); }); }); @@ -683,44 +579,39 @@ describe('Anthropic integration', () => { .expect({ transaction: { transaction: 'main', - spans: expect.arrayContaining([ - // First call: Last message is large and gets truncated (only C's remain, D's are cropped) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - // Messages should be present (truncation happened) and should be a JSON array - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/), - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - // Second call: Last message is small and kept without truncation - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - // Small message should be kept intact - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { role: 'user', content: 'This is a small message that fits within the limit' }, - ]), - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - ]), + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] First call: last message is large and truncated (only C's remain, D's cropped) + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"C+"\}\]$/, + ); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + + // [1] Second call: last message is small and kept without truncation + const smallMsgValue = JSON.stringify([ + { role: 'user', content: 'This is a small message that fits within the limit' }, + ]); + expect(secondSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe(smallMsgValue); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); }, }) .start() @@ -731,43 +622,43 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-media-truncation.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('truncates media attachment, keeping all other details', async () => { + const expectedMediaMessages = JSON.stringify([ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: '[Blob substitute]', + }, + }, + ], + }, + ]); await createRunner() .ignore('event') .expect({ transaction: { transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2, - // Only the last message (with filtered media) should be kept - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { - role: 'user', - content: [ - { - type: 'image', - source: { - type: 'base64', - media_type: 'image/png', - data: '[Blob substitute]', - }, - }, - ], - }, - ]), - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - status: 'ok', - }), - ]), + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] messages.create with media attachment — image data replaced, other fields preserved + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe(expectedMediaMessages); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(2); }, }) .start() @@ -781,20 +672,21 @@ describe('Anthropic integration', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('extracts system instructions from messages', async () => { + const expectedInstructions = JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]); await createRunner() .ignore('event') .expect({ transaction: { transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([ - { type: 'text', content: 'You are a helpful assistant' }, - ]), - }), - }), - ]), + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] messages.create — system instructions extracted into dedicated attribute + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE].value).toBe(expectedInstructions); }, }) .start() @@ -808,26 +700,6 @@ describe('Anthropic integration', () => { const EXPECTED_TRANSACTION_NO_TRUNCATION = { transaction: 'main', - spans: expect.arrayContaining([ - // Multiple messages should all be preserved (no popping to last message only) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { role: 'user', content: longContent }, - { role: 'assistant', content: 'Some reply' }, - { role: 'user', content: 'Follow-up question' }, - ]), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - }), - }), - // Long string input should not be truncated (messagesFromParams wraps it in an array) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([longStringInput]), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - }), - }), - ]), }; createEsmAndCjsTests( @@ -836,9 +708,29 @@ describe('Anthropic integration', () => { 'instrument-no-truncation.mjs', (createRunner, test) => { test('does not truncate input messages when enableTruncation is false', async () => { + const expectedAllMessages = JSON.stringify([ + { role: 'user', content: longContent }, + { role: 'assistant', content: 'Some reply' }, + { role: 'user', content: 'Follow-up question' }, + ]); + const expectedLongString = JSON.stringify([longStringInput]); await createRunner() .ignore('event') .expect({ transaction: EXPECTED_TRANSACTION_NO_TRUNCATION }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] messages.create with multi-message conversation — all messages preserved + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe(expectedAllMessages); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + + // [1] messages.create with long string input — not truncated + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe(expectedLongString); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(1); + }, + }) .start() .completed(); }); @@ -852,12 +744,25 @@ describe('Anthropic integration', () => { await createRunner() .expect({ span: container => { - const spans = container.items; - - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), - ); - expect(chatSpan).toBeDefined(); + expect(container.items).toHaveLength(8); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan, eighthSpan] = + container.items; + + // [0]–[5] express middleware + http.server spans + expect(firstSpan!.name).toBe('query'); + expect(secondSpan!.name).toBe('expressInit'); + expect(thirdSpan!.name).toBe('jsonParser'); + expect(fourthSpan!.name).toBe('/anthropic/v1/messages'); + expect(fifthSpan!.name).toBe('POST /anthropic/v1/messages'); + expect(sixthSpan!.name).toBe('POST'); + + // [6] messages.create — gen_ai.chat span with full (non-truncated) long content + expect(seventhSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain(streamingLongContent); + + // [7] root 'main' function span + expect(eighthSpan!.name).toBe('main'); }, }) .start() @@ -874,17 +779,30 @@ describe('Anthropic integration', () => { await createRunner() .expect({ span: container => { - const spans = container.items; - - // With explicit enableTruncation: true, content should be truncated despite streaming. - // Find the chat span by matching the start of the truncated content (the 'A' repeated messages). - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + expect(container.items).toHaveLength(8); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan, eighthSpan] = + container.items; + + // [0]–[5] express middleware + http.server spans + expect(firstSpan!.name).toBe('query'); + expect(secondSpan!.name).toBe('expressInit'); + expect(thirdSpan!.name).toBe('jsonParser'); + expect(fourthSpan!.name).toBe('/anthropic/v1/messages'); + expect(fifthSpan!.name).toBe('POST /anthropic/v1/messages'); + expect(sixthSpan!.name).toBe('POST'); + + // [6] messages.create — gen_ai.chat span whose content is truncated despite streaming + expect(seventhSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"AAAA/, ); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( streamingLongContent.length, ); + + // [7] root 'main' function span + expect(eighthSpan!.name).toBe('main'); }, }) .start() From 0474787851d0484ff5c251853b7ceb15f1d23ef9 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 09/18] Update Anthropic Cloudflare tests Co-Authored-By: Claude Opus 4.6 --- .../suites/tracing/anthropic-ai/test.ts | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts index 17cea5dbf95b..4f60868cddfb 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts @@ -1,4 +1,3 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import { expect, it } from 'vitest'; import { GEN_AI_OPERATION_NAME_ATTRIBUTE, @@ -21,30 +20,36 @@ it('traces a basic message creation request', async ({ signal }) => { const runner = createRunner(__dirname) .ignore('event') .expect(envelope => { + // Transaction item (first item in envelope) const transactionEvent = envelope[1]?.[0]?.[1] as any; - expect(transactionEvent.transaction).toBe('GET /'); - expect(transactionEvent.spans).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'msg_mock123', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - }), - description: 'chat claude-3-haiku-20240307', - op: 'gen_ai.chat', - origin: 'auto.ai.anthropic', - }), - ]), - ); + + // Span container item (second item in same envelope) + const container = envelope[1]?.[1]?.[1] as any; + expect(container).toBeDefined(); + + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] chat claude-3-haiku-20240307 + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(firstSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.anthropic' }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'anthropic' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'claude-3-haiku-20240307', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'claude-3-haiku-20240307', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'msg_mock123' }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 }); }) .start(signal); await runner.makeRequest('get', '/'); From 2630a242d656c56f054a48e663ccc55137b42f9b Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 10/18] Update OpenAI Node tests Co-Authored-By: Claude Opus 4.6 --- .../tracing/openai/openai-tool-calls/test.ts | 548 +++-- .../suites/tracing/openai/test.ts | 1784 ++++++++++------- .../suites/tracing/openai/v6/test.ts | 1198 +++++++---- 3 files changed, 2202 insertions(+), 1328 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts index c66f5cb65c6b..a300a0a79192 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts @@ -76,212 +76,158 @@ describe('OpenAI Tool Calls integration', () => { }, ]); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - chat completion with tools (non-streaming) - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: WEATHER_TOOL_DEFINITION, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-tools-123', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["tool_calls"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - chat completion with tools and streaming - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: WEATHER_TOOL_DEFINITION, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-stream-tools-123', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["tool_calls"]', - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Third span - responses API with tools (non-streaming) - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: WEATHER_TOOL_DEFINITION, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_tools_789', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["completed"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Fourth span - responses API with tools and streaming - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: WEATHER_TOOL_DEFINITION, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_stream_tools_789', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["in_progress","completed"]', - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - chat completion with tools (non-streaming) with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather like in Paris today?"}]', - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: WEATHER_TOOL_DEFINITION, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-tools-123', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["tool_calls"]', - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: '[""]', - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: CHAT_TOOL_CALLS, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - chat completion with tools and streaming with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather like in Paris today?"}]', - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: WEATHER_TOOL_DEFINITION, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-stream-tools-123', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["tool_calls"]', - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: CHAT_STREAM_TOOL_CALLS, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Third span - responses API with tools (non-streaming) with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather like in Paris today?"}]', - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: WEATHER_TOOL_DEFINITION, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_tools_789', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["completed"]', - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: RESPONSES_TOOL_CALLS, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Fourth span - responses API with tools and streaming with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather like in Paris today?"}]', - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: WEATHER_TOOL_DEFINITION, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_stream_tools_789', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["in_progress","completed"]', - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: RESPONSES_TOOL_CALLS, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai tool calls related spans with sendDefaultPii: false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] chat completion with tools (non-streaming) + expect(firstSpan!.name).toBe('chat gpt-4'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: WEATHER_TOOL_DEFINITION, + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-tools-123', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["tool_calls"]', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 25 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 40 }); + + // [1] chat completion with tools and streaming + expect(secondSpan!.name).toBe('chat gpt-4'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: WEATHER_TOOL_DEFINITION, + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-tools-123', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["tool_calls"]', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 25, + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 40 }); + + // [2] responses API with tools (non-streaming) + expect(thirdSpan!.name).toBe('chat gpt-4'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: WEATHER_TOOL_DEFINITION, + }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_tools_789', + }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 12 }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 }); + + // [3] responses API with tools and streaming + expect(fourthSpan!.name).toBe('chat gpt-4'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: WEATHER_TOOL_DEFINITION, + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_stream_tools_789', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 12, + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 }); + }, + }) .start() .completed(); }); @@ -291,7 +237,203 @@ describe('OpenAI Tool Calls integration', () => { test('creates openai tool calls related spans with sendDefaultPii: true', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] chat completion with tools (non-streaming) with PII + expect(firstSpan!.name).toBe('chat gpt-4'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the weather like in Paris today?"}]', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: WEATHER_TOOL_DEFINITION, + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-tools-123', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["tool_calls"]', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toEqual({ type: 'string', value: '[""]' }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: CHAT_TOOL_CALLS, + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 25 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 40 }); + + // [1] chat completion with tools and streaming with PII + expect(secondSpan!.name).toBe('chat gpt-4'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the weather like in Paris today?"}]', + }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: WEATHER_TOOL_DEFINITION, + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-tools-123', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["tool_calls"]', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: CHAT_STREAM_TOOL_CALLS, + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 25, + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 40 }); + + // [2] responses API with tools (non-streaming) with PII + expect(thirdSpan!.name).toBe('chat gpt-4'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the weather like in Paris today?"}]', + }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: WEATHER_TOOL_DEFINITION, + }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_tools_789', + }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: RESPONSES_TOOL_CALLS, + }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 12 }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 }); + + // [3] responses API with tools and streaming with PII + expect(fourthSpan!.name).toBe('chat gpt-4'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the weather like in Paris today?"}]', + }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: WEATHER_TOOL_DEFINITION, + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_stream_tools_789', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]).toEqual({ + type: 'string', + value: RESPONSES_TOOL_CALLS, + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 12, + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 }); + }, + }) .start() .completed(); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index e3ecc4f80ae0..acc7442efbba 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -29,297 +29,193 @@ describe('OpenAI integration', () => { cleanupChildProcesses(); }); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_CHAT = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - basic chat completion without PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }, - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - responses API - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_mock456', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["completed"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 5, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 13, - }, - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Third span - error handling - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }, - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - // Fourth span - chat completions streaming - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.8, - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-stream-123', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 18, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Fifth span - responses API streaming - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_stream_456', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["in_progress","completed"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 6, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 16, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Sixth span - error handling in streaming context - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - }, - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_CHAT = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - basic chat completion with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the capital of France?"}]', - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([ - { type: 'text', content: 'You are a helpful assistant.' }, - ]), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["stop"]', - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: '["Hello from OpenAI mock!"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }, - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - responses API with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: 'Translate this to French: Hello', - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Response to: Translate this to French: Hello', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["completed"]', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_mock456', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 5, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 13, - }, - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Third span - error handling with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"This will fail"}]', - }, - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - // Fourth span - chat completions streaming with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.8, - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Tell me about streaming"}]', - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([ - { type: 'text', content: 'You are a helpful assistant.' }, - ]), - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Hello from OpenAI streaming!', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["stop"]', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-stream-123', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 18, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Fifth span - responses API streaming with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: 'Test streaming responses API', - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: - 'Streaming response to: Test streaming responses APITest streaming responses API', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["in_progress","completed"]', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_stream_456', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 6, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 16, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Sixth span - error handling in streaming context with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"This will fail"}]', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - }, - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_WITH_OPTIONS = { - transaction: 'main', - spans: expect.arrayContaining([ - // Check that custom options are respected - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response text when recordOutputs: true - }), - }), - // Check that custom options are respected for streaming - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response text when recordOutputs: true - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, // Should be marked as stream - }), - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with sendDefaultPii: false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_CHAT }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(6); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan] = container.items; + + // [0] basic chat completion without PII + expect(firstSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 25 }); + + // [1] responses API + expect(secondSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 5 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 13 }); + + // [2] error handling (non-streaming) + expect(thirdSpan!.name).toBe('chat error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + + // [3] chat completions streaming + expect(fourthSpan!.name).toBe('chat gpt-4'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 12 }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 18, + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 30 }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + + // [4] responses API streaming + expect(fifthSpan!.name).toBe('chat gpt-4'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(fifthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fifthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fifthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fifthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fifthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 6 }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 16 }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + + // [5] error handling in streaming context + expect(sixthSpan!.name).toBe('chat error-model'); + expect(sixthSpan!.status).toBe('error'); + expect(sixthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(sixthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(sixthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(sixthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + }, + }) .start() .completed(); }); @@ -329,7 +225,261 @@ describe('OpenAI integration', () => { test('creates openai related spans with sendDefaultPii: true', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_CHAT }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(6); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan] = container.items; + + // [0] basic chat completion with PII + expect(firstSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the capital of France?"}]', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant.' }]), + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["Hello from OpenAI mock!"]', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 25 }); + + // [1] responses API with PII + expect(secondSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Translate this to French: Hello', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Response to: Translate this to French: Hello', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 5 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 13 }); + + // [2] error handling with PII (non-streaming) + expect(thirdSpan!.name).toBe('chat error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); + + // [3] chat completions streaming with PII + expect(fourthSpan!.name).toBe('chat gpt-4'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"Tell me about streaming"}]', + }); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant.' }]), + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Hello from OpenAI streaming!', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 12 }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 18, + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 30 }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + + // [4] responses API streaming with PII + expect(fifthSpan!.name).toBe('chat gpt-4'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(fifthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fifthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fifthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fifthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fifthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fifthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(fifthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Test streaming responses API', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Streaming response to: Test streaming responses APITest streaming responses API', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 6 }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 16 }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + + // [5] error handling in streaming context with PII + expect(sixthSpan!.name).toBe('chat error-model'); + expect(sixthSpan!.status).toBe('error'); + expect(sixthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(sixthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(sixthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); + expect(sixthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(sixthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(sixthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + }, + }) .start() .completed(); }); @@ -339,7 +489,35 @@ describe('OpenAI integration', () => { test('creates openai related spans with custom options', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_WITH_OPTIONS }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(6); + const [firstSpan, , , fourthSpan] = container.items; + + // [0] non-streaming with input messages recorded via custom options + expect(firstSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + + // [3] streaming with input messages recorded via custom options + expect(fourthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + }, + }) .start() .completed(); }); @@ -347,30 +525,6 @@ describe('OpenAI integration', () => { const longContent = 'A'.repeat(50_000); - const EXPECTED_TRANSACTION_NO_TRUNCATION = { - transaction: 'main', - spans: expect.arrayContaining([ - // Multiple messages should all be preserved (no popping to last message only) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { role: 'user', content: longContent }, - { role: 'assistant', content: 'Some reply' }, - { role: 'user', content: 'Follow-up question' }, - ]), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - }), - }), - // Responses API long string input should not be truncated or wrapped in quotes - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: 'B'.repeat(50_000), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - }), - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario-no-truncation.mjs', @@ -379,115 +533,153 @@ describe('OpenAI integration', () => { test('does not truncate input messages when enableTruncation is false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_NO_TRUNCATION }) + .expect({ + transaction: { + transaction: 'main', + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] chat completions: multiple messages all preserved (no popping to last message only) + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: JSON.stringify([ + { role: 'user', content: longContent }, + { role: 'assistant', content: 'Some reply' }, + { role: 'user', content: 'Follow-up question' }, + ]), + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toMatchObject({ + type: 'integer', + value: 3, + }); + + // [1] responses API long string input is not truncated or wrapped in quotes + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: 'B'.repeat(50_000), + }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toMatchObject({ + type: 'integer', + value: 1, + }); + }, + }) .start() .completed(); }); }, ); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_EMBEDDINGS = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - embeddings API - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]: 'float', - [GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]: 1536, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 10, - }, - description: 'embeddings text-embedding-3-small', - op: 'gen_ai.embeddings', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - embeddings API error model - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }, - description: 'embeddings error-model', - op: 'gen_ai.embeddings', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_EMBEDDINGS = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - embeddings API with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]: 'float', - [GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]: 1536, - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: 'Embedding test!', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 10, - }, - description: 'embeddings text-embedding-3-small', - op: 'gen_ai.embeddings', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - embeddings API error model with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: 'Error embedding test!', - }, - description: 'embeddings error-model', - op: 'gen_ai.embeddings', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - // Third span - embeddings API with multiple inputs (this does not get truncated) - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: '["First input text","Second input text","Third input text"]', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 10, - }, - description: 'embeddings text-embedding-3-small', - op: 'gen_ai.embeddings', - origin: 'auto.ai.openai', - status: 'ok', - }), - ]), - }; createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with sendDefaultPii: false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_EMBEDDINGS }) + .expect({ + transaction: { + transaction: 'main', + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] embeddings API (single input) + expect(firstSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + + // [1] embeddings API error model + expect(secondSpan!.name).toBe('embeddings error-model'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + + // [2] embeddings API (multiple inputs) + expect(thirdSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + }, + }) .start() .completed(); }); @@ -497,7 +689,112 @@ describe('OpenAI integration', () => { test('creates openai related spans with sendDefaultPii: true', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_EMBEDDINGS }) + .expect({ + transaction: { + transaction: 'main', + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] embeddings API with PII (single input) + expect(firstSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(firstSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Embedding test!', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + + // [1] embeddings API error model with PII + expect(secondSpan!.name).toBe('embeddings error-model'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(secondSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Error embedding test!', + }); + + // [2] embeddings API with multiple inputs (not truncated) + expect(thirdSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(thirdSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["First input text","Second input text","Third input text"]', + }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + }, + }) .start() .completed(); }); @@ -598,50 +895,74 @@ describe('OpenAI integration', () => { .expect({ transaction: { transaction: 'main', - spans: expect.arrayContaining([ - // First call: Last message is large and gets truncated (only C's remain, D's are cropped) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - // Messages should be present (truncation happened) and should be a JSON array of a single index - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2, - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.stringMatching( - /^\[\{"type":"text","content":"A+"\}\]$/, - ), - }), - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second call: Last message is small and kept without truncation - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - // Small message should be kept intact - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { role: 'user', content: 'This is a small message that fits within the limit' }, - ]), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2, - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.stringMatching( - /^\[\{"type":"text","content":"A+"\}\]$/, - ), - }), - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - ]), + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] Last message is large and gets truncated (only C's remain, D's are cropped) + expect(firstSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 2, + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"C+"\}\]$/, + ); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE].value).toMatch( + /^\[\{"type":"text","content":"A+"\}\]$/, + ); + + // [1] Last message is small and kept without truncation + expect(secondSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: JSON.stringify([ + { role: 'user', content: 'This is a small message that fits within the limit' }, + ]), + }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 2, + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE].value).toMatch( + /^\[\{"type":"text","content":"A+"\}\]$/, + ); }, }) .start() @@ -661,24 +982,35 @@ describe('OpenAI integration', () => { .expect({ transaction: { transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - // Messages should be present and should include truncated string input (contains only As) - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^A+$/), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - }), - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - ]), + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] long A-string input is truncated + expect(firstSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch(/^A+$/); }, }) .start() @@ -688,201 +1020,151 @@ describe('OpenAI integration', () => { ); // Test for conversation ID support (Conversations API and previous_response_id) - const EXPECTED_TRANSACTION_CONVERSATION = { - transaction: 'conversation-test', - spans: expect.arrayContaining([ - // First span - conversations.create returns conversation object with id - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - // The conversation ID should be captured from the response - [GEN_AI_CONVERSATION_ID_ATTRIBUTE]: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', - }), - description: 'chat unknown', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - responses.create with conversation parameter - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - // The conversation ID should be captured from the request - [GEN_AI_CONVERSATION_ID_ATTRIBUTE]: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', - }), - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Third span - responses.create without conversation (first in chain, should NOT have gen_ai.conversation.id) - expect.objectContaining({ - data: expect.not.objectContaining({ - [GEN_AI_CONVERSATION_ID_ATTRIBUTE]: expect.anything(), - }), - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Fourth span - responses.create with previous_response_id (chaining) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - // The previous_response_id should be captured as conversation.id - [GEN_AI_CONVERSATION_ID_ATTRIBUTE]: 'resp_mock_conv_123', - }), - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario-conversation.mjs', 'instrument.mjs', (createRunner, test) => { test('captures conversation ID from Conversations API and previous_response_id', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_CONVERSATION }) + .expect({ + transaction: { + transaction: 'conversation-test', + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] conversations.create returns conversation object with id + expect(firstSpan!.name).toBe('chat unknown'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', + }); + + // [1] responses.create with conversation parameter + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(secondSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', + }); + + // [2] responses.create without conversation (first in chain, should NOT have gen_ai.conversation.id) + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(thirdSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE]).toBeUndefined(); + + // [3] responses.create with previous_response_id (chaining) + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fourthSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_mock_conv_123', + }); + }, + }) .start() .completed(); }); }); // Test for manual conversation ID setting using setConversationId() - const EXPECTED_TRANSACTION_MANUAL_CONVERSATION_ID = { - transaction: 'chat-with-manual-conversation-id', - spans: expect.arrayContaining([ - // All three chat completion spans should have the same manually-set conversation ID - expect.objectContaining({ - data: expect.objectContaining({ - 'gen_ai.conversation.id': 'user_chat_session_abc123', - 'gen_ai.system': 'openai', - 'gen_ai.request.model': 'gpt-4', - 'gen_ai.operation.name': 'chat', - 'sentry.op': 'gen_ai.chat', - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - expect.objectContaining({ - data: expect.objectContaining({ - 'gen_ai.conversation.id': 'user_chat_session_abc123', - 'gen_ai.system': 'openai', - 'gen_ai.request.model': 'gpt-4', - 'gen_ai.operation.name': 'chat', - 'sentry.op': 'gen_ai.chat', - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - expect.objectContaining({ - data: expect.objectContaining({ - 'gen_ai.conversation.id': 'user_chat_session_abc123', - 'gen_ai.system': 'openai', - 'gen_ai.request.model': 'gpt-4', - 'gen_ai.operation.name': 'chat', - 'sentry.op': 'gen_ai.chat', - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario-manual-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('attaches manual conversation ID set via setConversationId() to all chat spans', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_MANUAL_CONVERSATION_ID }) + .expect({ + transaction: { + transaction: 'chat-with-manual-conversation-id', + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // All three chat completion spans should have the same manually-set conversation ID + for (const span of [firstSpan, secondSpan, thirdSpan]) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes['gen_ai.conversation.id']).toEqual({ + type: 'string', + value: 'user_chat_session_abc123', + }); + expect(span!.attributes['gen_ai.system']).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes['gen_ai.operation.name']).toEqual({ type: 'string', value: 'chat' }); + expect(span!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } + }, + }) .start() .completed(); }); }); - // Test for scope isolation - different scopes have different conversation IDs - const EXPECTED_TRANSACTION_CONVERSATION_1 = { - transaction: 'GET /chat/conversation-1', - spans: expect.arrayContaining([ - // Both chat completion spans in conversation 1 should have conv_user1_session_abc - expect.objectContaining({ - data: expect.objectContaining({ - 'gen_ai.conversation.id': 'conv_user1_session_abc', - 'gen_ai.system': 'openai', - 'gen_ai.request.model': 'gpt-4', - 'sentry.op': 'gen_ai.chat', - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - expect.objectContaining({ - data: expect.objectContaining({ - 'gen_ai.conversation.id': 'conv_user1_session_abc', - 'gen_ai.system': 'openai', - 'gen_ai.request.model': 'gpt-4', - 'sentry.op': 'gen_ai.chat', - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - ]), - }; - - const EXPECTED_TRANSACTION_CONVERSATION_2 = { - transaction: 'GET /chat/conversation-2', - spans: expect.arrayContaining([ - // Both chat completion spans in conversation 2 should have conv_user2_session_xyz - expect.objectContaining({ - data: expect.objectContaining({ - 'gen_ai.conversation.id': 'conv_user2_session_xyz', - 'gen_ai.system': 'openai', - 'gen_ai.request.model': 'gpt-4', - 'sentry.op': 'gen_ai.chat', - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - expect.objectContaining({ - data: expect.objectContaining({ - 'gen_ai.conversation.id': 'conv_user2_session_xyz', - 'gen_ai.system': 'openai', - 'gen_ai.request.model': 'gpt-4', - 'sentry.op': 'gen_ai.chat', - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario-separate-scope-1.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 1', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_CONVERSATION_1 }) + .expect({ + transaction: { + transaction: 'GET /chat/conversation-1', + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // Both chat completion spans should have the expected conversation ID + for (const span of [firstSpan, secondSpan]) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes['gen_ai.conversation.id']).toEqual({ + type: 'string', + value: 'conv_user1_session_abc', + }); + expect(span!.attributes['gen_ai.system']).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } + }, + }) .start() .completed(); }); @@ -892,7 +1174,30 @@ describe('OpenAI integration', () => { test('isolates conversation IDs across separate scopes - conversation 2', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_CONVERSATION_2 }) + .expect({ + transaction: { + transaction: 'GET /chat/conversation-2', + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // Both chat completion spans should have the expected conversation ID + for (const span of [firstSpan, secondSpan]) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes['gen_ai.conversation.id']).toEqual({ + type: 'string', + value: 'conv_user2_session_xyz', + }); + expect(span!.attributes['gen_ai.system']).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } + }, + }) .start() .completed(); }); @@ -909,15 +1214,18 @@ describe('OpenAI integration', () => { .expect({ transaction: { transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([ - { type: 'text', content: 'You are a helpful assistant' }, - ]), - }), - }), - ]), + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] chat completion with system instructions extracted from messages + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), + }); }, }) .start() @@ -933,30 +1241,24 @@ describe('OpenAI integration', () => { .expect({ transaction: { transaction: 'main', - spans: expect.arrayContaining([ - // First call using .withResponse() - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-withresponse', - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - status: 'ok', - }), - // Second call using .asResponse() - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-withresponse', - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - status: 'ok', - }), - ]), + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // Both calls should produce spans with the same response ID + for (const span of [firstSpan, secondSpan]) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-withresponse', + }); + } }, }) .start() @@ -971,32 +1273,25 @@ describe('OpenAI integration', () => { .expect({ transaction: { transaction: 'main', - spans: expect.arrayContaining([ - // Single image vision request - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4o', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringContaining('[Blob substitute]'), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - }), - description: 'chat gpt-4o', - op: 'gen_ai.chat', - status: 'ok', - }), - // Multiple images vision request - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4o', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringContaining('[Blob substitute]'), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - }), - description: 'chat gpt-4o', - op: 'gen_ai.chat', - status: 'ok', - }), - ]), + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // Both vision request spans should contain [Blob substitute] + for (const span of [firstSpan, secondSpan]) { + expect(span!.name).toBe('chat gpt-4o'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4o' }); + expect(span!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(span!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain('[Blob substitute]'); + } }, }) .start() @@ -1009,14 +1304,17 @@ describe('OpenAI integration', () => { .expect({ transaction: { transaction: 'main', - spans: expect.arrayContaining([ - // The second span (multiple images) should still contain the https URL - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringContaining('https://example.com/image.png'), - }), - }), - ]), + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + const [, secondSpan] = container.items; + + // [1] multiple images span contains the https URL + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain( + 'https://example.com/image.png', + ); }, }) .start() @@ -1032,17 +1330,24 @@ describe('OpenAI integration', () => { await createRunner() .expect({ span: container => { - const spans = container.items; + expect(container.items).toHaveLength(15); + const [, , , , , , seventhSpan, , , , , , , fourteenthSpan] = container.items; - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), - ); - expect(chatSpan).toBeDefined(); + // [6] chat completions: full long content preserved (streaming disables truncation) + expect(seventhSpan!.name).toBe('chat gpt-4'); + expect(seventhSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain(streamingLongContent); - const responsesSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongString), - ); - expect(responsesSpan).toBeDefined(); + // [13] responses API: full long string preserved (streaming disables truncation) + expect(fourteenthSpan!.name).toBe('chat gpt-4'); + expect(fourteenthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(fourteenthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain(streamingLongString); }, }) .start() @@ -1059,24 +1364,31 @@ describe('OpenAI integration', () => { await createRunner() .expect({ span: container => { - const spans = container.items; + expect(container.items).toHaveLength(15); + const [, , , , , , seventhSpan, , , , , , , fourteenthSpan] = container.items; - // With explicit enableTruncation: true, content should be truncated despite streaming. + // [6] chat completions: content truncated despite streaming when enableTruncation is explicitly true. // Truncation keeps only the last message (50k 'A's) and crops it to the byte limit. - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + expect(seventhSpan!.name).toBe('chat gpt-4'); + expect(seventhSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"AAAA/, ); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( streamingLongContent.length, ); - // The responses API string input (50k 'B's) should also be truncated. - const responsesSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('BBB'), - ); - expect(responsesSpan).toBeDefined(); - expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( + // [13] responses API: long string input (50k 'B's) is also truncated. + expect(fourteenthSpan!.name).toBe('chat gpt-4'); + expect(fourteenthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(fourteenthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch(/^BBB/); + expect(fourteenthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( streamingLongString.length, ); }, diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts index b282282305eb..906cb5ee61c4 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts @@ -28,390 +28,6 @@ describe('OpenAI integration (V6)', () => { cleanupChildProcesses(); }); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_CHAT = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - basic chat completion without PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }, - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - responses API - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_mock456', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["completed"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 5, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 13, - }, - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Third span - error handling - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }, - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - // Fourth span - chat completions streaming - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.8, - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-stream-123', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 18, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Fifth span - responses API streaming - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_stream_456', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["in_progress","completed"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 6, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 16, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - }, - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Sixth span - error handling in streaming context - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - }, - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_CHAT = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - basic chat completion with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the capital of France?"}]', - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: '[{"type":"text","content":"You are a helpful assistant."}]', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["stop"]', - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: '["Hello from OpenAI mock!"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }, - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - responses API with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: 'Translate this to French: Hello', - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Response to: Translate this to French: Hello', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["completed"]', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_mock456', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 5, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 13, - }, - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Third span - error handling with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"This will fail"}]', - }, - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - // Fourth span - chat completions streaming with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.8, - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"Tell me about streaming"}]', - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: '[{"type":"text","content":"You are a helpful assistant."}]', - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Hello from OpenAI streaming!', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["stop"]', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-stream-123', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 18, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Fifth span - responses API streaming with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: 'Test streaming responses API', - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: - 'Streaming response to: Test streaming responses APITest streaming responses API', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["in_progress","completed"]', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'resp_stream_456', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 6, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 16, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - }), - description: 'chat gpt-4', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Sixth span - error handling in streaming context with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"This will fail"}]', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - }, - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_WITH_OPTIONS = { - transaction: 'main', - spans: expect.arrayContaining([ - // Check that custom options are respected - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.any(String), // System instructions should be extracted - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response text when recordOutputs: true - }), - }), - // Check that custom options are respected for streaming - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.any(String), // System instructions should be extracted - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response text when recordOutputs: true - [GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true, // Should be marked as stream - }), - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_EMBEDDINGS = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - embeddings API - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]: 'float', - [GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]: 1536, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 10, - }, - description: 'embeddings text-embedding-3-small', - op: 'gen_ai.embeddings', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - embeddings API error model - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }, - description: 'embeddings error-model', - op: 'gen_ai.embeddings', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_EMBEDDINGS = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - embeddings API with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]: 'float', - [GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]: 1536, - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: 'Embedding test!', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 10, - }, - description: 'embeddings text-embedding-3-small', - op: 'gen_ai.embeddings', - origin: 'auto.ai.openai', - status: 'ok', - }), - // Second span - embeddings API error model with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: 'Error embedding test!', - }, - description: 'embeddings error-model', - op: 'gen_ai.embeddings', - origin: 'auto.ai.openai', - status: 'internal_error', - }), - // Third span - embeddings API with multiple inputs (this does not get truncated) - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: '["First input text","Second input text","Third input text"]', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 10, - }, - description: 'embeddings text-embedding-3-small', - op: 'gen_ai.embeddings', - origin: 'auto.ai.openai', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario-chat.mjs', @@ -420,7 +36,240 @@ describe('OpenAI integration (V6)', () => { test('creates openai related spans with sendDefaultPii: false (v6)', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_CHAT }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(6); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan] = container.items; + + // [0] basic chat completion without PII + expect(firstSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ + type: 'double', + value: 0.7, + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 15, + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 25, + }); + + // [1] responses API + expect(secondSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 5, + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 8, + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 13, + }); + + // [2] error handling (non-streaming) + expect(thirdSpan!.name).toBe('chat error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + + // [3] chat completions streaming + expect(fourthSpan!.name).toBe('chat gpt-4'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 12, + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 18, + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 30, + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + + // [4] responses API streaming + expect(fifthSpan!.name).toBe('chat gpt-4'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(fifthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fifthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fifthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fifthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fifthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 6, + }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 16, + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + + // [5] error handling in streaming context + expect(sixthSpan!.name).toBe('chat error-model'); + expect(sixthSpan!.status).toBe('error'); + expect(sixthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(sixthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(sixthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(sixthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + }, + }) .start() .completed(); }); @@ -440,7 +289,312 @@ describe('OpenAI integration (V6)', () => { test('creates openai related spans with sendDefaultPii: true (v6)', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_CHAT }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(6); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan] = container.items; + + // [0] basic chat completion with PII + expect(firstSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ + type: 'double', + value: 0.7, + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the capital of France?"}]', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"type":"text","content":"You are a helpful assistant."}]', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["Hello from OpenAI mock!"]', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 15, + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 25, + }); + + // [1] responses API with PII + expect(secondSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Translate this to French: Hello', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Response to: Translate this to French: Hello', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 5, + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 8, + }); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 13, + }); + + // [2] error handling with PII (non-streaming) + expect(thirdSpan!.name).toBe('chat error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); + + // [3] chat completions streaming with PII + expect(fourthSpan!.name).toBe('chat gpt-4'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fourthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"Tell me about streaming"}]', + }); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"type":"text","content":"You are a helpful assistant."}]', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Hello from OpenAI streaming!', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 12, + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 18, + }); + expect(fourthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 30, + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + + // [4] responses API streaming with PII + expect(fifthSpan!.name).toBe('chat gpt-4'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(fifthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(fifthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(fifthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(fifthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(fifthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fifthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(fifthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Test streaming responses API', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Streaming response to: Test streaming responses APITest streaming responses API', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 6, + }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + expect(fifthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 16, + }); + expect(fifthSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + + // [5] error handling in streaming context with PII + expect(sixthSpan!.name).toBe('chat error-model'); + expect(sixthSpan!.status).toBe('error'); + expect(sixthSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(sixthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(sixthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(sixthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); + expect(sixthSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(sixthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(sixthSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + }, + }) .start() .completed(); }); @@ -460,7 +614,51 @@ describe('OpenAI integration (V6)', () => { test('creates openai related spans with custom options (v6)', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_WITH_OPTIONS }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(6); + const [firstSpan, , , fourthSpan] = container.items; + + // [0] non-streaming with input messages recorded via custom options + expect(firstSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toMatchObject({ + type: 'integer', + value: 1, + }); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + + // [3] streaming with input messages recorded via custom options + expect(fourthSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ type: 'boolean', value: true }); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toMatchObject({ + type: 'integer', + value: 1, + }); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(fourthSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + }, + }) .start() .completed(); }); @@ -480,7 +678,112 @@ describe('OpenAI integration (V6)', () => { test('creates openai related spans with sendDefaultPii: false (v6)', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_EMBEDDINGS }) + .expect({ + transaction: { + transaction: 'main', + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] embeddings API (single input) + expect(firstSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + + // [1] embeddings API error model + expect(secondSpan!.name).toBe('embeddings error-model'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + + // [2] embeddings API (multiple inputs) + expect(thirdSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + }, + }) .start() .completed(); }); @@ -500,7 +803,124 @@ describe('OpenAI integration (V6)', () => { test('creates openai related spans with sendDefaultPii: true (v6)', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_EMBEDDINGS }) + .expect({ + transaction: { + transaction: 'main', + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] embeddings API with PII (single input) + expect(firstSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(firstSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(firstSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Embedding test!', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + + // [1] embeddings API error model with PII + expect(secondSpan!.name).toBe('embeddings error-model'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(secondSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(secondSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Error embedding test!', + }); + + // [2] embeddings API with multiple inputs (not truncated) + expect(thirdSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(thirdSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(thirdSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["First input text","Second input text","Third input text"]', + }); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + expect(thirdSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 10, + }); + }, + }) .start() .completed(); }); From 7eea49496710d949346b259fd3c553f62a326669 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 11/18] Update OpenAI Cloudflare tests Co-Authored-By: Claude Opus 4.6 --- .../suites/tracing/openai/test.ts | 60 +++++++++++-------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts index 1c057e1a986c..76288214e9f2 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts @@ -1,4 +1,3 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import { expect, it } from 'vitest'; import { GEN_AI_OPERATION_NAME_ATTRIBUTE, @@ -23,32 +22,41 @@ it('traces a basic chat completion request', async ({ signal }) => { const runner = createRunner(__dirname) .ignore('event') .expect(envelope => { - const transactionEvent = envelope[1]?.[0]?.[1]; - + // Transaction item (first item in envelope) + const transactionEvent = envelope[1]?.[0]?.[1] as any; expect(transactionEvent.transaction).toBe('GET /'); - expect(transactionEvent.spans).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-mock123', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["stop"]', - }), - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - }), - ]), - ); + + // Span container item (second item in same envelope) + const container = envelope[1]?.[1]?.[1] as any; + expect(container).toBeDefined(); + + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] chat gpt-3.5-turbo + expect(firstSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(firstSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.openai' }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'gpt-3.5-turbo' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 25 }); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["stop"]', + }); }) .start(signal); await runner.makeRequest('get', '/'); From 5d2ec5a41510265336543eb7900efd5fd9ed2ea5 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 12/18] Update LangChain Node tests Co-Authored-By: Claude Opus 4.6 --- .../suites/tracing/langchain/test.ts | 796 ++++++++---------- .../suites/tracing/langchain/v1/test.ts | 537 +++++------- 2 files changed, 580 insertions(+), 753 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index f85e3187ac78..8c1087fcba4b 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -1,8 +1,7 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { - GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE, + GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, GEN_AI_OPERATION_NAME_ATTRIBUTE, @@ -29,146 +28,56 @@ describe('LangChain integration', () => { cleanupChildProcesses(); }); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - chat model with claude-3-5-sonnet - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: expect.any(String), - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Second span - chat model with claude-3-opus - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-opus-20240229', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.9, - [GEN_AI_REQUEST_TOP_P_ATTRIBUTE]: 0.95, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 200, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: expect.any(String), - }), - description: 'chat claude-3-opus-20240229', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Third span - error handling - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }), - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - chat model with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response when recordOutputs: true - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: expect.any(String), - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Second span - chat model with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-opus-20240229', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.9, - [GEN_AI_REQUEST_TOP_P_ATTRIBUTE]: 0.95, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 200, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response when recordOutputs: true - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: expect.any(String), - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }), - description: 'chat claude-3-opus-20240229', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Third span - error handling with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - }), - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'internal_error', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates langchain related spans with sendDefaultPii: false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] chat model with claude-3-5-sonnet + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); + + // [1] chat model with claude-3-opus + expect(secondSpan!.name).toBe('chat claude-3-opus-20240229'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-opus-20240229'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.9); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE].value).toBe(0.95); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(200); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + + // [2] error handling + expect(thirdSpan!.name).toBe('chat error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(thirdSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-model'); + }, + }) .start() .completed(); }); @@ -176,14 +85,23 @@ describe('LangChain integration', () => { test('does not create duplicate spans from double module patching', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: event => { - const spans = event.spans || []; - const genAiChatSpans = spans.filter(span => span.op === 'gen_ai.chat'); + span: container => { // The scenario makes 3 LangChain calls (2 successful + 1 error). // Without the dedup guard, the file-level and module-level hooks // both patch the same prototype, producing 6 spans instead of 3. - expect(genAiChatSpans).toHaveLength(3); + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] chat claude-3-5-sonnet + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + + // [1] chat claude-3-opus + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + + // [2] chat error-model + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); }, }) .start() @@ -195,107 +113,88 @@ describe('LangChain integration', () => { test('creates langchain related spans with sendDefaultPii: true', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] chat model with PII + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + + // [1] chat model with PII + expect(secondSpan!.name).toBe('chat claude-3-opus-20240229'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-opus-20240229'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.9); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE].value).toBe(0.95); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(200); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + + // [2] error handling with PII + expect(thirdSpan!.name).toBe('chat error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-model'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + }, + }) .start() .completed(); }); }); - const EXPECTED_TRANSACTION_TOOL_CALLS = { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 150, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 50, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: 'tool_use', - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: expect.any(String), - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument.mjs', (createRunner, test) => { test('creates langchain spans with tool calls', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_TOOL_CALLS }).start().completed(); + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] chat with tool_use stop reason + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(150); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(20); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(30); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(50); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('tool_use'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]).toBeDefined(); + }, + }) + .start() + .completed(); }); }); - const EXPECTED_TRANSACTION_MESSAGE_TRUNCATION = { - transaction: 'main', - spans: expect.arrayContaining([ - // First call: String input truncated (only C's remain, D's are cropped) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - // Messages should be present and should include truncated string input (contains only Cs) - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/), - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Second call: Array input, last message truncated (only C's remain, D's are cropped) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2, - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.any(String), - // Messages should be present (truncation happened) and should be a JSON array of a single index (contains only Cs) - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/), - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Third call: Last message is small and kept without truncation - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2, - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.any(String), - // Small message should be kept intact - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { role: 'user', content: 'This is a small message that fits within the limit' }, - ]), - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario-message-truncation.mjs', @@ -304,7 +203,36 @@ describe('LangChain integration', () => { test('truncates messages when they exceed byte limit', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_MESSAGE_TRUNCATION }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] String input truncated (only C's remain, D's are cropped) + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(1); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"C+"\}\]$/, + ); + + // [1] Array input, last message truncated (only C's remain, D's are cropped) + expect(secondSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(2); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"C+"\}\]$/, + ); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]).toBeDefined(); + + // [2] Last message is small and kept without truncation + expect(thirdSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(2); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + JSON.stringify([{ role: 'user', content: 'This is a small message that fits within the limit' }]), + ); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]).toBeDefined(); + }, + }) .start() .completed(); }); @@ -319,46 +247,25 @@ describe('LangChain integration', () => { test('demonstrates timing issue with duplicate spans (ESM only)', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: event => { - // This test highlights the limitation: if a user creates an Anthropic client - // before importing LangChain, that client will still be instrumented and - // could cause duplicate spans when used alongside LangChain. - - const spans = event.spans || []; - - // First call: Direct Anthropic call made BEFORE LangChain import - // This should have Anthropic instrumentation (origin: 'auto.ai.anthropic') - const firstAnthropicSpan = spans.find( - span => span.description === 'chat claude-3-5-sonnet-20241022' && span.origin === 'auto.ai.anthropic', - ); - - // Second call: LangChain call - // This should have LangChain instrumentation (origin: 'auto.ai.langchain') - const langchainSpan = spans.find( - span => span.description === 'chat claude-3-5-sonnet-20241022' && span.origin === 'auto.ai.langchain', - ); - - // Third call: Direct Anthropic call made AFTER LangChain import - // This should NOT have Anthropic instrumentation (skip works correctly) - // Count how many Anthropic spans we have - should be exactly 1 - const anthropicSpans = spans.filter( - span => span.description === 'chat claude-3-5-sonnet-20241022' && span.origin === 'auto.ai.anthropic', - ); + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; // Verify the edge case limitation: - // - First Anthropic client (created before LangChain) IS instrumented - expect(firstAnthropicSpan).toBeDefined(); - expect(firstAnthropicSpan?.origin).toBe('auto.ai.anthropic'); - - // - LangChain call IS instrumented by LangChain - expect(langchainSpan).toBeDefined(); - expect(langchainSpan?.origin).toBe('auto.ai.langchain'); - - // - Second Anthropic client (created after LangChain) is NOT instrumented - // This demonstrates that the skip mechanism works for NEW clients - // We should only have ONE Anthropic span (the first one), not two - expect(anthropicSpans).toHaveLength(1); + // [0] Direct Anthropic call made BEFORE LangChain import — IS instrumented + // by Anthropic (origin: 'auto.ai.anthropic'). + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); + + // [1] LangChain call — IS instrumented by LangChain (origin: 'auto.ai.langchain'). + expect(secondSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + + // Third call (not present): Direct Anthropic call made AFTER LangChain import + // is NOT instrumented, which demonstrates the skip mechanism works for NEW + // clients. We should only have ONE Anthropic span (the first one), not two. }, }) .start() @@ -377,18 +284,18 @@ describe('LangChain integration', () => { test('extracts system instructions from messages', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([ - { type: 'text', content: 'You are a helpful assistant' }, - ]), - }), - }), - ]), + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] chat with extracted system instructions + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE].value).toBe( + JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), + ); }, }) .start() @@ -401,32 +308,32 @@ describe('LangChain integration', () => { test('uses runName for chain spans instead of unknown_chain', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - description: 'chain format_prompt', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langchain', - data: expect.objectContaining({ - 'langchain.chain.name': 'format_prompt', - }), - }), - expect.objectContaining({ - description: 'chain parse_output', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langchain', - data: expect.objectContaining({ - 'langchain.chain.name': 'parse_output', - }), - }), - expect.objectContaining({ - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - }), - ]), + span: container => { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] format_prompt chain (invoke_agent) + expect(firstSpan!.name).toBe('chain format_prompt'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes['langchain.chain.name'].value).toBe('format_prompt'); + + // [1] chat model invoked inside the chain + expect(secondSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + + // [2] parse_output chain (invoke_agent) + expect(thirdSpan!.name).toBe('chain parse_output'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(thirdSpan!.attributes['langchain.chain.name'].value).toBe('parse_output'); + + // [3] unknown_chain (fallback name) + expect(fourthSpan!.name).toBe('chain unknown_chain'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); }, }) .start() @@ -438,101 +345,60 @@ describe('LangChain integration', () => { // Embeddings tests // ========================================================================= - const EXPECTED_TRANSACTION_EMBEDDINGS = { - transaction: 'main', - spans: expect.arrayContaining([ - // embedQuery span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]: 1536, - }), - description: 'embeddings text-embedding-3-small', - op: GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, - origin: 'auto.ai.langchain', - status: 'ok', - }), - // embedDocuments span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - }), - description: 'embeddings text-embedding-3-small', - op: GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Error span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }), - description: 'embeddings error-model', - op: GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, - origin: 'auto.ai.langchain', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_EMBEDDINGS_PII = { - transaction: 'main', - spans: expect.arrayContaining([ - // embedQuery span with input recorded - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-3-small', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: 'Hello world', - }), - description: 'embeddings text-embedding-3-small', - op: GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, - origin: 'auto.ai.langchain', - status: 'ok', - }), - // embedDocuments span with input recorded - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: JSON.stringify(['First document', 'Second document']), - }), - description: 'embeddings text-embedding-3-small', - op: GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE, - origin: 'auto.ai.langchain', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates embedding spans with sendDefaultPii: false', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_EMBEDDINGS }).start().completed(); + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] embedQuery span + expect(firstSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('embeddings'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('openai'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('text-embedding-3-small'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE].value).toBe(1536); + + // [1] embedDocuments span + expect(secondSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE); + + // [2] Error span + expect(thirdSpan!.name).toBe('embeddings error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('openai'); + }, + }) + .start() + .completed(); }); test('does not create duplicate embedding spans from double module patching', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: event => { - const spans = event.spans || []; - const embeddingSpans = spans.filter(span => span.op === GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE); + span: container => { // The scenario makes 3 embedding calls (2 successful + 1 error). - expect(embeddingSpans).toHaveLength(3); + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] embedQuery + expect(firstSpan!.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE); + + // [1] embedDocuments + expect(secondSpan!.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE); + + // [2] error embedding call + expect(thirdSpan!.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS_OPERATION_ATTRIBUTE); }, }) .start() @@ -544,7 +410,30 @@ describe('LangChain integration', () => { test('creates embedding spans with sendDefaultPii: true', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_EMBEDDINGS_PII }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] embedQuery span with input recorded + expect(firstSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE].value).toBe('Hello world'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE].value).toBe(1536); + + // [1] embedDocuments span with input recorded + expect(secondSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE].value).toBe( + JSON.stringify(['First document', 'Second document']), + ); + + // [2] error embedding span (input still recorded with PII) + expect(thirdSpan!.name).toBe('embeddings error-model'); + expect(thirdSpan!.status).toBe('error'); + }, + }) .start() .completed(); }); @@ -552,22 +441,6 @@ describe('LangChain integration', () => { const longContent = 'A'.repeat(50_000); - const EXPECTED_TRANSACTION_NO_TRUNCATION = { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { role: 'user', content: longContent }, - { role: 'assistant', content: 'Some reply' }, - { role: 'user', content: 'Follow-up question' }, - ]), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - }), - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario-no-truncation.mjs', @@ -576,7 +449,24 @@ describe('LangChain integration', () => { test('does not truncate input messages when enableTruncation is false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_NO_TRUNCATION }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] chat with full (untruncated) input messages + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + JSON.stringify([ + { role: 'user', content: longContent }, + { role: 'assistant', content: 'Some reply' }, + { role: 'user', content: 'Follow-up question' }, + ]), + ); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + }, + }) .start() .completed(); }); @@ -590,12 +480,41 @@ describe('LangChain integration', () => { await createRunner() .expect({ span: container => { - const spans = container.items; + expect(container.items).toHaveLength(8); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan, eighthSpan] = + container.items; - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), - ); - expect(chatSpan).toBeDefined(); + // [0] express middleware — query + expect(firstSpan!.name).toBe('query'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [1] express middleware — expressInit + expect(secondSpan!.name).toBe('expressInit'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [2] express middleware — jsonParser + expect(thirdSpan!.name).toBe('jsonParser'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [3] express route handler — /v1/messages + expect(fourthSpan!.name).toBe('/v1/messages'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('request_handler.express'); + + // [4] express http.server — POST /v1/messages + expect(fifthSpan!.name).toBe('POST /v1/messages'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('http.server'); + + // [5] outbound HTTP client — POST + expect(sixthSpan!.name).toBe('POST'); + + // [6] LangChain chat span — carries the full (untruncated) input messages + expect(seventhSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain(streamingLongContent); + + // [7] main — root span (streamed alongside) + expect(eighthSpan!.name).toBe('main'); + expect(eighthSpan!.attributes['sentry.op'].value).toBe('function'); }, }) .start() @@ -612,16 +531,47 @@ describe('LangChain integration', () => { await createRunner() .expect({ span: container => { - const spans = container.items; - - // With explicit enableTruncation: true, content should be truncated despite streaming. - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + expect(container.items).toHaveLength(8); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan, eighthSpan] = + container.items; + + // [0] express middleware — query + expect(firstSpan!.name).toBe('query'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [1] express middleware — expressInit + expect(secondSpan!.name).toBe('expressInit'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [2] express middleware — jsonParser + expect(thirdSpan!.name).toBe('jsonParser'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [3] express route handler — /v1/messages + expect(fourthSpan!.name).toBe('/v1/messages'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('request_handler.express'); + + // [4] express http.server — POST /v1/messages + expect(fifthSpan!.name).toBe('POST /v1/messages'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('http.server'); + + // [5] outbound HTTP client — POST + expect(sixthSpan!.name).toBe('POST'); + + // [6] LangChain chat span — content truncated despite streaming + // (explicit enableTruncation: true). + expect(seventhSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"AAAA/, ); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( streamingLongContent.length, ); + + // [7] main — root span + expect(eighthSpan!.name).toBe('main'); + expect(eighthSpan!.attributes['sentry.op'].value).toBe('function'); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index 032e33c75dfd..309d165fd674 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -1,4 +1,3 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import { afterAll, expect } from 'vitest'; import { GEN_AI_INPUT_MESSAGES_ATTRIBUTE, @@ -29,141 +28,6 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { cleanupChildProcesses(); }); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - chat model with claude-3-5-sonnet - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: expect.any(String), - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Second span - chat model with claude-3-opus - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-opus-20240229', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.9, - [GEN_AI_REQUEST_TOP_P_ATTRIBUTE]: 0.95, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 200, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: expect.any(String), - }), - description: 'chat claude-3-opus-20240229', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Third span - error handling - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }), - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - chat model with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response when recordOutputs: true - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: expect.any(String), - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Second span - chat model with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-opus-20240229', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.9, - [GEN_AI_REQUEST_TOP_P_ATTRIBUTE]: 0.95, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 200, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response when recordOutputs: true - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: expect.any(String), - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }), - description: 'chat claude-3-opus-20240229', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Third span - error handling with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - }), - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'internal_error', - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario.mjs', @@ -172,7 +36,52 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { test('creates langchain related spans with sendDefaultPii: false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] chat model with claude-3-5-sonnet + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); + + // [1] chat model with claude-3-opus + expect(secondSpan!.name).toBe('chat claude-3-opus-20240229'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-opus-20240229'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.9); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE].value).toBe(0.95); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(200); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + + // [2] error handling + expect(thirdSpan!.name).toBe('chat error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(thirdSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-model'); + }, + }) .start() .completed(); }); @@ -194,7 +103,52 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { test('creates langchain related spans with sendDefaultPii: true', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] chat model with PII + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + + // [1] chat model with PII + expect(secondSpan!.name).toBe('chat claude-3-opus-20240229'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-opus-20240229'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.9); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE].value).toBe(0.95); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(200); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); + + // [2] error handling with PII + expect(thirdSpan!.name).toBe('chat error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-model'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + }, + }) .start() .completed(); }); @@ -208,34 +162,6 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { }, ); - const EXPECTED_TRANSACTION_TOOL_CALLS = { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 150, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 30, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 50, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: 'tool_use', - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: expect.any(String), - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario-tools.mjs', @@ -244,7 +170,28 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { test('creates langchain spans with tool calls', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_TOOL_CALLS }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] chat with tool_use stop reason + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(150); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(20); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(30); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(50); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('tool_use'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]).toBeDefined(); + }, + }) .start() .completed(); }); @@ -258,68 +205,6 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { }, ); - const EXPECTED_TRANSACTION_MESSAGE_TRUNCATION = { - transaction: 'main', - spans: expect.arrayContaining([ - // First call: String input truncated (only C's remain, D's are cropped) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 1, - // Messages should be present and should include truncated string input (contains only Cs) - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/), - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Second call: Array input, last message truncated (only C's remain, D's are cropped) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2, - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.stringMatching(/^\[\{"type":"text","content":"A+"\}\]$/), - // Messages should be present (truncation happened) and should be a JSON array of a single index (contains only Cs) - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/), - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Third call: Last message is small and kept without truncation - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2, - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.stringMatching(/^\[\{"type":"text","content":"A+"\}\]$/), - - // Small message should be kept intact - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { role: 'user', content: 'This is a small message that fits within the limit' }, - ]), - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario-message-truncation.mjs', @@ -328,7 +213,40 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { test('truncates messages when they exceed byte limit', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_MESSAGE_TRUNCATION }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] String input truncated (only C's remain, D's are cropped) + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(1); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"C+"\}\]$/, + ); + + // [1] Array input, last message truncated (only C's remain, D's are cropped) + expect(secondSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(2); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"C+"\}\]$/, + ); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE].value).toMatch( + /^\[\{"type":"text","content":"A+"\}\]$/, + ); + + // [2] Last message is small and kept without truncation + expect(thirdSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(2); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + JSON.stringify([{ role: 'user', content: 'This is a small message that fits within the limit' }]), + ); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE].value).toMatch( + /^\[\{"type":"text","content":"A+"\}\]$/, + ); + }, + }) .start() .completed(); }); @@ -350,46 +268,23 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { test('demonstrates timing issue with duplicate spans (ESM only)', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: event => { - // This test highlights the limitation: if a user creates an Anthropic client - // before importing LangChain, that client will still be instrumented and - // could cause duplicate spans when used alongside LangChain. + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; - const spans = event.spans || []; + // [0] Direct Anthropic call made BEFORE LangChain import — instrumented + // by Anthropic (origin: 'auto.ai.anthropic'). + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); - // First call: Direct Anthropic call made BEFORE LangChain import - // This should have Anthropic instrumentation (origin: 'auto.ai.anthropic') - const firstAnthropicSpan = spans.find( - span => span.description === 'chat claude-3-5-sonnet-20241022' && span.origin === 'auto.ai.anthropic', - ); - - // Second call: LangChain call - // This should have LangChain instrumentation (origin: 'auto.ai.langchain') - const langchainSpan = spans.find( - span => span.description === 'chat claude-3-5-sonnet-20241022' && span.origin === 'auto.ai.langchain', - ); + // [1] LangChain call — instrumented by LangChain (origin: 'auto.ai.langchain'). + expect(secondSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - // Third call: Direct Anthropic call made AFTER LangChain import - // This should NOT have Anthropic instrumentation (skip works correctly) - // Count how many Anthropic spans we have - should be exactly 1 - const anthropicSpans = spans.filter( - span => span.description === 'chat claude-3-5-sonnet-20241022' && span.origin === 'auto.ai.anthropic', - ); - - // Verify the edge case limitation: - // - First Anthropic client (created before LangChain) IS instrumented - expect(firstAnthropicSpan).toBeDefined(); - expect(firstAnthropicSpan?.origin).toBe('auto.ai.anthropic'); - - // - LangChain call IS instrumented by LangChain - expect(langchainSpan).toBeDefined(); - expect(langchainSpan?.origin).toBe('auto.ai.langchain'); - - // - Second Anthropic client (created after LangChain) is NOT instrumented - // This demonstrates that the skip mechanism works for NEW clients - // We should only have ONE Anthropic span (the first one), not two - expect(anthropicSpans).toHaveLength(1); + // Third call (not present): Direct Anthropic call made AFTER LangChain import + // is NOT instrumented, demonstrating the skip mechanism works for NEW clients. }, }) .start() @@ -406,69 +301,6 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { }, ); - const EXPECTED_TRANSACTION_INIT_CHAT_MODEL = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - initChatModel with gpt-4o - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-4o', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4o', - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: 'stop', - }), - description: 'chat gpt-4o', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Second span - initChatModel with gpt-3.5-turbo - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.5, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: expect.any(String), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]: 'stop', - }), - description: 'chat gpt-3.5-turbo', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'ok', - }), - // Third span - error handling - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }), - description: 'chat error-model', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - status: 'internal_error', - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario-init-chat-model.mjs', @@ -477,7 +309,52 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { test('creates langchain spans using initChatModel with OpenAI', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_INIT_CHAT_MODEL }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] initChatModel with gpt-4o + expect(firstSpan!.name).toBe('chat gpt-4o'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('openai'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('gpt-4o'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(8); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(12); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(20); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('gpt-4o'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); + + // [1] initChatModel with gpt-3.5-turbo + expect(secondSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('openai'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('gpt-3.5-turbo'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.5); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(8); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(12); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(20); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('gpt-3.5-turbo'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); + + // [2] error handling + expect(thirdSpan!.name).toBe('chat error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(thirdSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('openai'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-model'); + }, + }) .start() .completed(); }); From 9968bc8f0a378c079c9aae8055bafc649446df02 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 13/18] Update LangChain Cloudflare tests Co-Authored-By: Claude Opus 4.6 --- .../suites/tracing/langchain/test.ts | 84 +++++++++---------- 1 file changed, 38 insertions(+), 46 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts index d4abc4ae7220..d05255f2c437 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts @@ -1,4 +1,3 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import { expect, it } from 'vitest'; import { GEN_AI_OPERATION_NAME_ATTRIBUTE, @@ -22,53 +21,46 @@ it('traces langchain chat model, chain, and tool invocations', async ({ signal } const runner = createRunner(__dirname) .ignore('event') .expect(envelope => { + // Transaction item (first item in envelope) const transactionEvent = envelope[1]?.[0]?.[1] as any; - expect(transactionEvent.transaction).toBe('GET /'); - expect(transactionEvent.spans).toEqual( - expect.arrayContaining([ - // Chat model span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25, - }), - description: 'chat claude-3-5-sonnet-20241022', - op: 'gen_ai.chat', - origin: 'auto.ai.langchain', - }), - // Chain span - expect.objectContaining({ - data: expect.objectContaining({ - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - 'langchain.chain.name': 'my_test_chain', - }), - description: 'chain my_test_chain', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langchain', - }), - // Tool span - expect.objectContaining({ - data: expect.objectContaining({ - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'search_tool', - }), - description: 'execute_tool search_tool', - op: 'gen_ai.execute_tool', - origin: 'auto.ai.langchain', - }), - ]), - ); + + // Span container item (second item in same envelope) + const container = envelope[1]?.[1]?.[1] as any; + expect(container).toBeDefined(); + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] chat claude-3-5-sonnet-20241022 + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(firstSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langchain' }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'anthropic' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'claude-3-5-sonnet-20241022', + }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 100 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 15 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 25 }); + + // [1] chain my_test_chain + expect(secondSpan!.name).toBe('chain my_test_chain'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langchain' }); + expect(secondSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.invoke_agent' }); + expect(secondSpan!.attributes['langchain.chain.name']).toEqual({ type: 'string', value: 'my_test_chain' }); + + // [2] execute_tool search_tool + expect(thirdSpan!.name).toBe('execute_tool search_tool'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langchain' }); + expect(thirdSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.execute_tool' }); + expect(thirdSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'search_tool' }); }) .start(signal); await runner.makeRequest('get', '/'); From 9e8767b7b6c43cdfe98b2d08a17cb571c15f2186 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 14/18] Update LangGraph Node tests Co-Authored-By: Claude Opus 4.6 --- .../suites/tracing/langgraph/test.ts | 791 ++++++++++-------- 1 file changed, 449 insertions(+), 342 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts index 0837efb63c2f..84a115599935 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts @@ -1,4 +1,3 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_AGENT_NAME_ATTRIBUTE, @@ -8,7 +7,6 @@ import { GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_PIPELINE_NAME_ATTRIBUTE, GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, - GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, GEN_AI_RESPONSE_TEXT_ATTRIBUTE, GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, @@ -24,271 +22,315 @@ describe('LangGraph integration', () => { cleanupChildProcesses(); }); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { - transaction: 'langgraph-test', - spans: expect.arrayContaining([ - // create_agent span - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'weather_assistant', - }, - description: 'create_agent weather_assistant', - op: 'gen_ai.create_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // First invoke_agent span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'weather_assistant', - [GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'weather_assistant', - }), - description: 'invoke_agent weather_assistant', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // Second invoke_agent span - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'weather_assistant', - [GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'weather_assistant', - }), - description: 'invoke_agent weather_assistant', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { - transaction: 'langgraph-test', - spans: expect.arrayContaining([ - // create_agent span (PII enabled doesn't affect this span) - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'weather_assistant', - }, - description: 'create_agent weather_assistant', - op: 'gen_ai.create_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // First invoke_agent span with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'weather_assistant', - [GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'weather_assistant', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringContaining('What is the weather today?'), - }), - description: 'invoke_agent weather_assistant', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // Second invoke_agent span with PII and multiple messages - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'weather_assistant', - [GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'weather_assistant', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringContaining('Tell me about the weather'), - }), - description: 'invoke_agent weather_assistant', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - ]), - }; - - const EXPECTED_TRANSACTION_WITH_TOOLS = { - transaction: 'langgraph-tools-test', - spans: expect.arrayContaining([ - // create_agent span for first graph (no tool calls) - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'tool_agent', - }, - description: 'create_agent tool_agent', - op: 'gen_ai.create_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // invoke_agent span with tools available but not called - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'tool_agent', - [GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'tool_agent', - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: expect.stringContaining('get_weather'), - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringContaining('What is the weather?'), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4-0613', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.stringContaining('Response without calling tools'), - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 25, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 40, - }), - description: 'invoke_agent tool_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // create_agent span for second graph (with tool calls) - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'tool_calling_agent', - }, - description: 'create_agent tool_calling_agent', - op: 'gen_ai.create_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // invoke_agent span with tool calls and execution - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'tool_calling_agent', - [GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'tool_calling_agent', - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: expect.stringContaining('get_weather'), - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringContaining('San Francisco'), - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-4-0613', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.stringMatching(/"role":"tool"/), - // Verify tool_calls are captured - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: expect.stringContaining('get_weather'), - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 80, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 40, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 120, - }), - description: 'invoke_agent tool_calling_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - ]), - }; - - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test, mode) => { test('should instrument LangGraph with default PII settings', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .expect({ transaction: { transaction: 'langgraph-test' } }) + .expect({ + span: container => { + // CJS instrumentation duplicates each LangGraph span (top-level + nested child). + if (mode === 'cjs') { + expect(container.items).toHaveLength(6); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan] = container.items; + + // [0] create_agent (top-level) + expect(firstSpan!.name).toBe('create_agent weather_assistant'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('create_agent'); + expect(firstSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('weather_assistant'); + + // [1] create_agent (nested duplicate) + expect(secondSpan!.name).toBe('create_agent weather_assistant'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + + // [2] first invoke_agent (top-level) + expect(thirdSpan!.name).toBe('invoke_agent weather_assistant'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('weather_assistant'); + expect(thirdSpan!.attributes[GEN_AI_PIPELINE_NAME_ATTRIBUTE].value).toBe('weather_assistant'); + + // [3] first invoke_agent (nested duplicate) + expect(fourthSpan!.name).toBe('invoke_agent weather_assistant'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + + // [4] second invoke_agent (top-level) + expect(fifthSpan!.name).toBe('invoke_agent weather_assistant'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(fifthSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('weather_assistant'); + + // [5] second invoke_agent (nested duplicate) + expect(sixthSpan!.name).toBe('invoke_agent weather_assistant'); + expect(sixthSpan!.status).toBe('ok'); + expect(sixthSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + } else { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] create_agent + expect(firstSpan!.name).toBe('create_agent weather_assistant'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('create_agent'); + expect(firstSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('weather_assistant'); + + // [1] first invoke_agent + expect(secondSpan!.name).toBe('invoke_agent weather_assistant'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('invoke_agent'); + expect(secondSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('weather_assistant'); + expect(secondSpan!.attributes[GEN_AI_PIPELINE_NAME_ATTRIBUTE].value).toBe('weather_assistant'); + + // [2] second invoke_agent + expect(thirdSpan!.name).toBe('invoke_agent weather_assistant'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('weather_assistant'); + expect(thirdSpan!.attributes[GEN_AI_PIPELINE_NAME_ATTRIBUTE].value).toBe('weather_assistant'); + } + }, + }) .start() .completed(); }); }); - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test, mode) => { test('should instrument LangGraph with sendDefaultPii: true', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }) + .expect({ transaction: { transaction: 'langgraph-test' } }) + .expect({ + span: container => { + if (mode === 'cjs') { + expect(container.items).toHaveLength(6); + const [firstSpan, , thirdSpan, , fifthSpan] = container.items; + + // [0] create_agent (top-level) + expect(firstSpan!.name).toBe('create_agent weather_assistant'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + + // [2] first invoke_agent with PII ("What is the weather today?") + expect(thirdSpan!.name).toBe('invoke_agent weather_assistant'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain( + 'What is the weather today?', + ); + + // [4] second invoke_agent with PII ("Tell me about the weather") + expect(fifthSpan!.name).toBe('invoke_agent weather_assistant'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(fifthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain( + 'Tell me about the weather', + ); + } else { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] create_agent + expect(firstSpan!.name).toBe('create_agent weather_assistant'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + + // [1] first invoke_agent with PII ("What is the weather today?") + expect(secondSpan!.name).toBe('invoke_agent weather_assistant'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain( + 'What is the weather today?', + ); + + // [2] second invoke_agent with PII ("Tell me about the weather") + expect(thirdSpan!.name).toBe('invoke_agent weather_assistant'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain( + 'Tell me about the weather', + ); + } + }, + }) .start() .completed(); }); }); - createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test, mode) => { test('should capture tools from LangGraph agent', { timeout: 30000 }, async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_WITH_TOOLS }).start().completed(); + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'langgraph-tools-test' } }) + .expect({ + span: container => { + if (mode === 'cjs') { + expect(container.items).toHaveLength(8); + const [firstSpan, , thirdSpan, , fifthSpan, , seventhSpan] = container.items; + + // [0] create_agent tool_agent (top-level) + expect(firstSpan!.name).toBe('create_agent tool_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + expect(firstSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('tool_agent'); + + // [2] invoke_agent tool_agent (tools available, not called) + expect(thirdSpan!.name).toBe('invoke_agent tool_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toContain('get_weather'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain('What is the weather?'); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('gpt-4-0613'); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toContain( + 'Response without calling tools', + ); + expect(thirdSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(25); + expect(thirdSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(thirdSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(40); + + // [4] create_agent tool_calling_agent (top-level) + expect(fifthSpan!.name).toBe('create_agent tool_calling_agent'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + expect(fifthSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('tool_calling_agent'); + + // [6] invoke_agent tool_calling_agent (with tool calls) + expect(seventhSpan!.name).toBe('invoke_agent tool_calling_agent'); + expect(seventhSpan!.status).toBe('ok'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain('San Francisco'); + expect(seventhSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('gpt-4-0613'); + expect(seventhSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toMatch(/"role":"tool"/); + expect(seventhSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE].value).toContain('get_weather'); + expect(seventhSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(80); + expect(seventhSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(40); + expect(seventhSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(120); + } else { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] create_agent tool_agent + expect(firstSpan!.name).toBe('create_agent tool_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + expect(firstSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('tool_agent'); + + // [1] invoke_agent tool_agent (tools available, not called) + expect(secondSpan!.name).toBe('invoke_agent tool_agent'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toContain('get_weather'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain('What is the weather?'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('gpt-4-0613'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toContain( + 'Response without calling tools', + ); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(25); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(40); + + // [2] create_agent tool_calling_agent + expect(thirdSpan!.name).toBe('create_agent tool_calling_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + expect(thirdSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('tool_calling_agent'); + + // [3] invoke_agent tool_calling_agent (with tool calls) + expect(fourthSpan!.name).toBe('invoke_agent tool_calling_agent'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain('San Francisco'); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('gpt-4-0613'); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE].value).toMatch(/"role":"tool"/); + expect(fourthSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE].value).toContain('get_weather'); + expect(fourthSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(80); + expect(fourthSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(40); + expect(fourthSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(120); + } + }, + }) + .start() + .completed(); }); }); // Test for thread_id (conversation ID) support - const EXPECTED_TRANSACTION_THREAD_ID = { - transaction: 'langgraph-thread-id-test', - spans: expect.arrayContaining([ - // create_agent span - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'thread_test_agent', - }, - description: 'create_agent thread_test_agent', - op: 'gen_ai.create_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // First invoke_agent span with thread_id - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'thread_test_agent', - [GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'thread_test_agent', - // The thread_id should be captured as conversation.id - [GEN_AI_CONVERSATION_ID_ATTRIBUTE]: 'thread_abc123_session_1', - }), - description: 'invoke_agent thread_test_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // Second invoke_agent span with different thread_id - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'thread_test_agent', - [GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'thread_test_agent', - // Different thread_id for different conversation - [GEN_AI_CONVERSATION_ID_ATTRIBUTE]: 'thread_xyz789_session_2', - }), - description: 'invoke_agent thread_test_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // Third invoke_agent span without thread_id (should NOT have gen_ai.conversation.id) - expect.objectContaining({ - data: expect.not.objectContaining({ - [GEN_AI_CONVERSATION_ID_ATTRIBUTE]: expect.anything(), - }), - description: 'invoke_agent thread_test_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - ]), - }; - - createEsmAndCjsTests(__dirname, 'scenario-thread-id.mjs', 'instrument.mjs', (createRunner, test) => { + createEsmAndCjsTests(__dirname, 'scenario-thread-id.mjs', 'instrument.mjs', (createRunner, test, mode) => { test('should capture thread_id as gen_ai.conversation.id', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_THREAD_ID }).start().completed(); + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'langgraph-thread-id-test' } }) + .expect({ + span: container => { + if (mode === 'cjs') { + expect(container.items).toHaveLength(8); + const [firstSpan, , thirdSpan, , fifthSpan, , seventhSpan] = container.items; + + // [0] create_agent (top-level) + expect(firstSpan!.name).toBe('create_agent thread_test_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + + // [2] first invoke_agent with thread_abc123_session_1 + expect(thirdSpan!.name).toBe('invoke_agent thread_test_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE].value).toBe('thread_abc123_session_1'); + + // [4] second invoke_agent with thread_xyz789_session_2 + expect(fifthSpan!.name).toBe('invoke_agent thread_test_agent'); + expect(fifthSpan!.status).toBe('ok'); + expect(fifthSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE].value).toBe('thread_xyz789_session_2'); + + // [6] third invoke_agent without thread_id + expect(seventhSpan!.name).toBe('invoke_agent thread_test_agent'); + expect(seventhSpan!.status).toBe('ok'); + expect(seventhSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE]).toBeUndefined(); + } else { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] create_agent + expect(firstSpan!.name).toBe('create_agent thread_test_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + + // [1] first invoke_agent with thread_abc123_session_1 + expect(secondSpan!.name).toBe('invoke_agent thread_test_agent'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE].value).toBe('thread_abc123_session_1'); + + // [2] second invoke_agent with thread_xyz789_session_2 + expect(thirdSpan!.name).toBe('invoke_agent thread_test_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE].value).toBe('thread_xyz789_session_2'); + + // [3] third invoke_agent without thread_id + expect(fourthSpan!.name).toBe('invoke_agent thread_test_agent'); + expect(fourthSpan!.status).toBe('ok'); + expect(fourthSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE]).toBeUndefined(); + } + }, + }) + .start() + .completed(); }); }); @@ -296,22 +338,32 @@ describe('LangGraph integration', () => { __dirname, 'scenario-system-instructions.mjs', 'instrument-with-pii.mjs', - (createRunner, test) => { + (createRunner, test, mode) => { test('extracts system instructions from messages', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([ - { type: 'text', content: 'You are a helpful assistant' }, - ]), - }), - }), - ]), + span: container => { + if (mode === 'cjs') { + expect(container.items).toHaveLength(4); + const [, , thirdSpan] = container.items; + + // [2] invoke_agent with system instructions (top-level) + expect(thirdSpan!.name).toBe('invoke_agent test-agent'); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE].value).toBe( + JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), + ); + } else { + expect(container.items).toHaveLength(2); + const [, secondSpan] = container.items; + + // [1] invoke_agent with system instructions + expect(secondSpan!.name).toBe('invoke_agent test-agent'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE].value).toBe( + JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), + ); + } }, }) .start() @@ -321,78 +373,104 @@ describe('LangGraph integration', () => { ); // Test for null input resume scenario - const EXPECTED_TRANSACTION_RESUME = { - transaction: 'langgraph-resume-test', - contexts: { - trace: expect.objectContaining({ - status: 'ok', - }), - }, - spans: expect.arrayContaining([ - // create_agent span - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'resume_agent', - }, - description: 'create_agent resume_agent', - op: 'gen_ai.create_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - // invoke_agent span with null input (resume) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'resume_agent', - [GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'resume_agent', - [GEN_AI_CONVERSATION_ID_ATTRIBUTE]: 'resume-thread-1', - }), - description: 'invoke_agent resume_agent', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', - status: 'ok', - }), - ]), - }; - - createEsmAndCjsTests(__dirname, 'scenario-resume.mjs', 'instrument.mjs', (createRunner, test) => { + createEsmAndCjsTests(__dirname, 'scenario-resume.mjs', 'instrument.mjs', (createRunner, test, mode) => { test('should not throw when invoke is called with null input (resume scenario)', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_RESUME }).start().completed(); + await createRunner() + .ignore('event') + .expect({ + transaction: { + transaction: 'langgraph-resume-test', + contexts: { + trace: expect.objectContaining({ + status: 'ok', + }), + }, + }, + }) + .expect({ + span: container => { + if (mode === 'cjs') { + expect(container.items).toHaveLength(6); + const [firstSpan, , thirdSpan] = container.items; + + // [0] create_agent resume_agent (top-level) + expect(firstSpan!.name).toBe('create_agent resume_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + expect(firstSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('resume_agent'); + + // [2] first invoke_agent with thread_id 'resume-thread-1' (top-level) + expect(thirdSpan!.name).toBe('invoke_agent resume_agent'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(thirdSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(thirdSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('resume_agent'); + expect(thirdSpan!.attributes[GEN_AI_PIPELINE_NAME_ATTRIBUTE].value).toBe('resume_agent'); + expect(thirdSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE].value).toBe('resume-thread-1'); + } else { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan] = container.items; + + // [0] create_agent resume_agent + expect(firstSpan!.name).toBe('create_agent resume_agent'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.create_agent'); + expect(firstSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('resume_agent'); + + // [1] first invoke_agent with thread_id 'resume-thread-1' + expect(secondSpan!.name).toBe('invoke_agent resume_agent'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(secondSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE].value).toBe('resume_agent'); + expect(secondSpan!.attributes[GEN_AI_PIPELINE_NAME_ATTRIBUTE].value).toBe('resume_agent'); + expect(secondSpan!.attributes[GEN_AI_CONVERSATION_ID_ATTRIBUTE].value).toBe('resume-thread-1'); + } + }, + }) + .start() + .completed(); }); }); const longContent = 'A'.repeat(50_000); - const EXPECTED_TRANSACTION_NO_TRUNCATION = { - transaction: 'langgraph-test', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { role: 'user', content: longContent }, - { role: 'assistant', content: 'Some reply' }, - { role: 'user', content: 'Follow-up question' }, - ]), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - }), - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario-no-truncation.mjs', 'instrument-no-truncation.mjs', - (createRunner, test) => { + (createRunner, test, mode) => { test('does not truncate input messages when enableTruncation is false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_NO_TRUNCATION }) + .expect({ transaction: { transaction: 'langgraph-test' } }) + .expect({ + span: container => { + const expectedMessages = JSON.stringify([ + { role: 'user', content: longContent }, + { role: 'assistant', content: 'Some reply' }, + { role: 'user', content: 'Follow-up question' }, + ]); + + if (mode === 'cjs') { + expect(container.items).toHaveLength(4); + const [, , thirdSpan] = container.items; + + // [2] invoke_agent with untruncated input (top-level) + expect(thirdSpan!.name).toBe('invoke_agent weather_assistant'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe(expectedMessages); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + } else { + expect(container.items).toHaveLength(2); + const [, secondSpan] = container.items; + + // [1] invoke_agent with untruncated input + expect(secondSpan!.name).toBe('invoke_agent weather_assistant'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe(expectedMessages); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + } + }, + }) .start() .completed(); }); @@ -401,43 +479,72 @@ describe('LangGraph integration', () => { const streamingLongContent = 'A'.repeat(50_000); - createEsmAndCjsTests(__dirname, 'scenario-span-streaming.mjs', 'instrument-streaming.mjs', (createRunner, test) => { - test('automatically disables truncation when span streaming is enabled', async () => { - await createRunner() - .expect({ - span: container => { - const spans = container.items; + createEsmAndCjsTests( + __dirname, + 'scenario-span-streaming.mjs', + 'instrument-streaming.mjs', + (createRunner, test, mode) => { + test('automatically disables truncation when span streaming is enabled', async () => { + await createRunner() + .expect({ + span: container => { + if (mode === 'cjs') { + expect(container.items).toHaveLength(5); + const [, , thirdSpan] = container.items; - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), - ); - expect(chatSpan).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }); + // [2] invoke_agent with untruncated streaming content (top-level) + expect(thirdSpan!.name).toBe('invoke_agent weather_assistant'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain(streamingLongContent); + } else { + expect(container.items).toHaveLength(3); + const [, secondSpan] = container.items; + + // [1] invoke_agent with untruncated streaming content + expect(secondSpan!.name).toBe('invoke_agent weather_assistant'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain(streamingLongContent); + } + }, + }) + .start() + .completed(); + }); + }, + ); createEsmAndCjsTests( __dirname, 'scenario-span-streaming.mjs', 'instrument-streaming-with-truncation.mjs', - (createRunner, test) => { + (createRunner, test, mode) => { test('respects explicit enableTruncation: true even when span streaming is enabled', async () => { await createRunner() .expect({ span: container => { - const spans = container.items; + if (mode === 'cjs') { + expect(container.items).toHaveLength(5); + const [, , thirdSpan] = container.items; - // With explicit enableTruncation: true, content should be truncated despite streaming. - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), - ); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( - streamingLongContent.length, - ); + // [2] invoke_agent with truncated streaming content (top-level) + expect(thirdSpan!.name).toBe('invoke_agent weather_assistant'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"AAAA/, + ); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( + streamingLongContent.length, + ); + } else { + expect(container.items).toHaveLength(3); + const [, secondSpan] = container.items; + + // [1] invoke_agent with truncated streaming content + expect(secondSpan!.name).toBe('invoke_agent weather_assistant'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","content":"AAAA/, + ); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( + streamingLongContent.length, + ); + } }, }) .start() From 6a62f0bb34bd2553d985330f05c7698f34800414 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 15/18] Update LangGraph Cloudflare tests Co-Authored-By: Claude Opus 4.6 --- .../suites/tracing/langgraph/test.ts | 82 ++++++++++--------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/langgraph/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/langgraph/test.ts index 6efa07164df5..4c8623654273 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/langgraph/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/langgraph/test.ts @@ -1,11 +1,9 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import { expect, it } from 'vitest'; import { GEN_AI_AGENT_NAME_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ATTRIBUTE, GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_PIPELINE_NAME_ATTRIBUTE, - GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, @@ -22,48 +20,56 @@ it('traces langgraph compile and invoke operations', async ({ signal }) => { const runner = createRunner(__dirname) .ignore('event') .expect(envelope => { + // Transaction item (first item in envelope) const transactionEvent = envelope[1]?.[0]?.[1] as any; - expect(transactionEvent.transaction).toBe('GET /'); - // Check create_agent span - const createAgentSpan = transactionEvent.spans.find((span: any) => span.op === 'gen_ai.create_agent'); - expect(createAgentSpan).toMatchObject({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.create_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'weather_assistant', - }, - description: 'create_agent weather_assistant', - op: 'gen_ai.create_agent', - origin: 'auto.ai.langgraph', - }); + // Span container item (second item in same envelope) + const container = envelope[1]?.[1]?.[1] as any; + expect(container).toBeDefined(); - // Check invoke_agent span - const invokeAgentSpan = transactionEvent.spans.find((span: any) => span.op === 'gen_ai.invoke_agent'); - expect(invokeAgentSpan).toMatchObject({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph', - [GEN_AI_AGENT_NAME_ATTRIBUTE]: 'weather_assistant', - [GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'weather_assistant', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather in SF?"}]', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 20, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30, - }), - description: 'invoke_agent weather_assistant', - op: 'gen_ai.invoke_agent', - origin: 'auto.ai.langgraph', + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] create_agent weather_assistant + expect(firstSpan!.name).toBe('create_agent weather_assistant'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'create_agent', + }); + expect(firstSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.create_agent' }); + expect(firstSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langgraph' }); + expect(firstSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'weather_assistant', }); - // Verify tools are captured - if (invokeAgentSpan.data[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]) { - expect(invokeAgentSpan.data[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toMatch(/get_weather/); - } + // [1] invoke_agent weather_assistant + expect(secondSpan!.name).toBe('invoke_agent weather_assistant'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.invoke_agent' }); + expect(secondSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.langgraph' }); + expect(secondSpan!.attributes[GEN_AI_AGENT_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'weather_assistant', + }); + expect(secondSpan!.attributes[GEN_AI_PIPELINE_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'weather_assistant', + }); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the weather in SF?"}]', + }); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ type: 'string', value: 'mock-model' }); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 10 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 30 }); }) .start(signal); await runner.makeRequest('get', '/'); From 0667e9d72ba33592de73e1392f3a032f15e7697f Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 16/18] Update Google GenAI Node tests Co-Authored-By: Claude Opus 4.6 --- .../suites/tracing/google-genai/test.ts | 955 ++++++++---------- 1 file changed, 397 insertions(+), 558 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts index 9839ef5fa2c0..d0c7803f6d4c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts @@ -1,10 +1,8 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ATTRIBUTE, GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, - GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE, @@ -29,144 +27,48 @@ describe('Google GenAI integration', () => { cleanupChildProcesses(); }); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { - transaction: 'main', - spans: expect.arrayContaining([ - // chat.sendMessage (should get model from context) - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-pro', // Should get from chat context - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }, - description: 'chat gemini-1.5-pro', - op: 'gen_ai.chat', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // models.generateContent - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-flash', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_TOP_P_ATTRIBUTE]: 0.9, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }, - description: 'generate_content gemini-1.5-flash', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // error handling - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }, - description: 'generate_content error-model', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { - transaction: 'main', - spans: expect.arrayContaining([ - // chat.sendMessage with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-pro', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include message when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response when recordOutputs: true - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }), - description: 'chat gemini-1.5-pro', - op: 'gen_ai.chat', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // models.generateContent with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-flash', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_TOP_P_ATTRIBUTE]: 0.9, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include contents when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response when recordOutputs: true - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }), - description: 'generate_content gemini-1.5-flash', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // error handling with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include contents when recordInputs: true - }), - description: 'generate_content error-model', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_WITH_OPTIONS = { - transaction: 'main', - spans: expect.arrayContaining([ - // Check that custom options are respected - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include messages when recordInputs: true - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response text when recordOutputs: true - }), - description: expect.not.stringContaining('stream-response'), // Non-streaming span - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates google genai related spans with sendDefaultPii: false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] chat.sendMessage (should get model from context) + expect(firstSpan!.name).toBe('chat gemini-1.5-pro'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.google_genai'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('google_genai'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('gemini-1.5-pro'); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(8); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(12); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(20); + + // [1] models.generateContent + expect(secondSpan!.name).toBe('generate_content gemini-1.5-flash'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('google_genai'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('gemini-1.5-flash'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE].value).toBe(0.9); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(8); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(12); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(20); + + // [2] error handling + expect(thirdSpan!.name).toBe('generate_content error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('google_genai'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-model'); + }, + }) .start() .completed(); }); @@ -176,7 +78,41 @@ describe('Google GenAI integration', () => { test('creates google genai related spans with sendDefaultPii: true', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] chat.sendMessage with PII + expect(firstSpan!.name).toBe('chat gemini-1.5-pro'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('google_genai'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('gemini-1.5-pro'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(8); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(12); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(20); + + // [1] models.generateContent with PII + expect(secondSpan!.name).toBe('generate_content gemini-1.5-flash'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE].value).toBe(0.9); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + + // [2] error handling with PII + expect(thirdSpan!.name).toBe('generate_content error-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + }, + }) .start() .completed(); }); @@ -186,7 +122,24 @@ describe('Google GenAI integration', () => { test('creates google genai related spans with custom options', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_WITH_OPTIONS }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] chat.sendMessage with custom options (PII enabled via recordInputs/recordOutputs) + expect(firstSpan!.name).toBe('chat gemini-1.5-pro'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + + // [1] models.generateContent with custom options + expect(secondSpan!.name).toBe('generate_content gemini-1.5-flash'); + + // [2] error handling with custom options + expect(thirdSpan!.name).toBe('generate_content error-model'); + }, + }) .start() .completed(); }); @@ -195,242 +148,106 @@ describe('Google GenAI integration', () => { const EXPECTED_AVAILABLE_TOOLS_JSON = '[{"name":"controlLight","parametersJsonSchema":{"type":"object","properties":{"brightness":{"type":"number"},"colorTemperature":{"type":"string"}},"required":["brightness","colorTemperature"]}}]'; - const EXPECTED_TRANSACTION_TOOLS = { - transaction: 'main', - spans: expect.arrayContaining([ - // Non-streaming with tools - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-2.0-flash-001', - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: EXPECTED_AVAILABLE_TOOLS_JSON, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include contents - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response text - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: expect.any(String), // Should include tool calls - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 23, - }), - description: 'generate_content gemini-2.0-flash-001', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // Streaming with tools - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-2.0-flash-001', - [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: EXPECTED_AVAILABLE_TOOLS_JSON, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include contents - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response text - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: expect.any(String), // Should include tool calls - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'mock-response-tools-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gemini-2.0-flash-001', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 22, - }), - description: 'generate_content gemini-2.0-flash-001', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // Without tools for comparison - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-2.0-flash-001', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include contents - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: expect.any(String), // Should include response text - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }), - description: 'generate_content gemini-2.0-flash-001', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-options.mjs', (createRunner, test) => { test('creates google genai related spans with tool calls', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_TOOLS }).start().completed(); + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] Non-streaming with tools + expect(firstSpan!.name).toBe('generate_content gemini-2.0-flash-001'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toBe( + EXPECTED_AVAILABLE_TOOLS_JSON, + ); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]).toBeUndefined(); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(8); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(23); + + // [1] Streaming with tools + expect(secondSpan!.name).toBe('generate_content gemini-2.0-flash-001'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toBe( + EXPECTED_AVAILABLE_TOOLS_JSON, + ); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('mock-response-tools-id'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('gemini-2.0-flash-001'); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(12); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(22); + + // [2] Without tools for comparison + expect(thirdSpan!.name).toBe('generate_content gemini-2.0-flash-001'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toBeUndefined(); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toBeDefined(); + expect(thirdSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(8); + expect(thirdSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(12); + expect(thirdSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(20); + }, + }) + .start() + .completed(); }); }); - const EXPECTED_TRANSACTION_STREAMING = { - transaction: 'main', - spans: expect.arrayContaining([ - // models.generateContentStream (streaming) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-flash', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_TOP_P_ATTRIBUTE]: 0.9, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'mock-response-streaming-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gemini-1.5-pro', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["STOP"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 22, - }), - description: 'generate_content gemini-1.5-flash', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // chat.sendMessageStream (streaming) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-pro', - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'mock-response-streaming-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gemini-1.5-pro', - }), - description: 'chat gemini-1.5-pro', - op: 'gen_ai.chat', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // blocked content streaming - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - }), - description: 'generate_content blocked-model', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'internal_error', - }), - // error handling for streaming - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - }), - description: 'generate_content error-model', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'internal_error', - }), - ]), - }; - - const EXPECTED_TRANSACTION_STREAMING_PII_TRUE = { - transaction: 'main', - spans: expect.arrayContaining([ - // models.generateContentStream (streaming) with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-flash', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_TOP_P_ATTRIBUTE]: 0.9, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include contents when recordInputs: true - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'mock-response-streaming-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gemini-1.5-pro', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["STOP"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 22, - }), - description: 'generate_content gemini-1.5-flash', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // chat.sendMessageStream (streaming) with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-pro', - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include message when recordInputs: true - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - [GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'mock-response-streaming-id', - [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gemini-1.5-pro', - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["STOP"]', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 22, - }), - description: 'chat gemini-1.5-pro', - op: 'gen_ai.chat', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // blocked content stream with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'blocked-model', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include contents when recordInputs: true - [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, - }), - description: 'generate_content blocked-model', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'internal_error', - }), - // error handling for streaming with PII - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include contents when recordInputs: true - }), - description: 'generate_content error-model', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'internal_error', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario-streaming.mjs', 'instrument.mjs', (createRunner, test) => { test('creates google genai streaming spans with sendDefaultPii: false', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_STREAMING }).start().completed(); + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] models.generateContentStream (streaming) + expect(firstSpan!.name).toBe('generate_content gemini-1.5-flash'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE].value).toBe(0.9); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('mock-response-streaming-id'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('gemini-1.5-pro'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE].value).toBe('["STOP"]'); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(12); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(22); + + // [1] chat.sendMessageStream (streaming) + expect(secondSpan!.name).toBe('chat gemini-1.5-pro'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('mock-response-streaming-id'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('gemini-1.5-pro'); + + // [2] blocked content streaming + expect(thirdSpan!.name).toBe('generate_content blocked-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + // [3] error handling for streaming + expect(fourthSpan!.name).toBe('generate_content error-model'); + expect(fourthSpan!.status).toBe('error'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + }, + }) + .start() + .completed(); }); }); @@ -438,7 +255,43 @@ describe('Google GenAI integration', () => { test('creates google genai streaming spans with sendDefaultPii: true', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_STREAMING_PII_TRUE }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(4); + const [firstSpan, secondSpan, thirdSpan, fourthSpan] = container.items; + + // [0] models.generateContentStream (streaming) with PII + expect(firstSpan!.name).toBe('generate_content gemini-1.5-flash'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE].value).toBe(0.9); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE].value).toBe('["STOP"]'); + + // [1] chat.sendMessageStream (streaming) with PII + expect(secondSpan!.name).toBe('chat gemini-1.5-pro'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(secondSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE].value).toBe('["STOP"]'); + + // [2] blocked content stream with PII + expect(thirdSpan!.name).toBe('generate_content blocked-model'); + expect(thirdSpan!.status).toBe('error'); + expect(thirdSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(thirdSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + + // [3] error handling for streaming with PII + expect(fourthSpan!.name).toBe('generate_content error-model'); + expect(fourthSpan!.status).toBe('error'); + expect(fourthSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); + expect(fourthSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + }, + }) .start() .completed(); }); @@ -452,52 +305,32 @@ describe('Google GenAI integration', () => { test('truncates messages when they exceed byte limit - keeps only last message and crops it', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: { - transaction: 'main', - spans: expect.arrayContaining([ - // First call: Last message is large and gets truncated (only C's remain, D's are cropped) - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-flash', - // Messages should be present (truncation happened) and should be a JSON array with parts - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching( - /^\[\{"role":"user","parts":\[\{"text":"C+"\}\]\}\]$/, - ), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - }), - description: 'generate_content gemini-1.5-flash', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // Second call: Last message is small and kept without truncation - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-flash', - // Small message should be kept intact - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { - role: 'user', - parts: [{ text: 'This is a small message that fits within the limit' }], - }, - ]), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - }), - description: 'generate_content gemini-1.5-flash', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - ]), + span: container => { + expect(container.items).toHaveLength(2); + const [firstSpan, secondSpan] = container.items; + + // [0] First call: Last message is large and gets truncated (only C's remain, D's are cropped) + expect(firstSpan!.name).toBe('generate_content gemini-1.5-flash'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","parts":\[\{"text":"C+"\}\]\}\]$/, + ); + + // [1] Second call: Last message is small and kept without truncation + expect(secondSpan!.name).toBe('generate_content gemini-1.5-flash'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + expect(secondSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + JSON.stringify([ + { + role: 'user', + parts: [{ text: 'This is a small message that fits within the limit' }], + }, + ]), + ); }, }) .start() @@ -514,18 +347,17 @@ describe('Google GenAI integration', () => { test('extracts system instructions from messages', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ - transaction: { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([ - { type: 'text', content: 'You are a helpful assistant' }, - ]), - }), - }), - ]), + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] generate_content with system instructions extracted + expect(firstSpan!.name).toBe('generate_content gemini-1.5-flash'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE].value).toBe( + JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), + ); }, }) .start() @@ -534,111 +366,36 @@ describe('Google GenAI integration', () => { }, ); - const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_EMBEDDINGS = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - embedContent with string contents - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-004', - }, - description: 'embeddings text-embedding-004', - op: 'gen_ai.embeddings', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // Second span - embedContent error model - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - }, - description: 'embeddings error-model', - op: 'gen_ai.embeddings', - origin: 'auto.ai.google_genai', - status: 'internal_error', - }), - // Third span - embedContent with array contents - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-004', - }, - description: 'embeddings text-embedding-004', - op: 'gen_ai.embeddings', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - ]), - }; - - const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_EMBEDDINGS = { - transaction: 'main', - spans: expect.arrayContaining([ - // First span - embedContent with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-004', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: 'What is the capital of France?', - }, - description: 'embeddings text-embedding-004', - op: 'gen_ai.embeddings', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - // Second span - embedContent error model with PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'error-model', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: 'This will fail', - }, - description: 'embeddings error-model', - op: 'gen_ai.embeddings', - origin: 'auto.ai.google_genai', - status: 'internal_error', - }), - // Third span - embedContent with array contents and PII - expect.objectContaining({ - data: { - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-004', - [GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]: - '[{"role":"user","parts":[{"text":"First input text"}]},{"role":"user","parts":[{"text":"Second input text"}]}]', - }, - description: 'embeddings text-embedding-004', - op: 'gen_ai.embeddings', - origin: 'auto.ai.google_genai', - status: 'ok', - }), - ]), - }; - createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates google genai embeddings spans with sendDefaultPii: false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_EMBEDDINGS }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] embedContent with string contents (no PII) + expect(firstSpan!.name).toBe('embeddings text-embedding-004'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.google_genai'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('google_genai'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('text-embedding-004'); + expect(firstSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE]).toBeUndefined(); + + // [1] embedContent error model + expect(secondSpan!.name).toBe('embeddings error-model'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + + // [2] embedContent with array contents (no PII) + expect(thirdSpan!.name).toBe('embeddings text-embedding-004'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + }, + }) .start() .completed(); }); @@ -648,7 +405,33 @@ describe('Google GenAI integration', () => { test('creates google genai embeddings spans with sendDefaultPii: true', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_EMBEDDINGS }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] embedContent with string contents and PII + expect(firstSpan!.name).toBe('embeddings text-embedding-004'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('google_genai'); + expect(firstSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE].value).toBe( + 'What is the capital of France?', + ); + + // [1] embedContent error model with PII + expect(secondSpan!.name).toBe('embeddings error-model'); + expect(secondSpan!.status).toBe('error'); + expect(secondSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE].value).toBe('This will fail'); + + // [2] embedContent with array contents and PII + expect(thirdSpan!.name).toBe('embeddings text-embedding-004'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE].value).toBe( + '[{"role":"user","parts":[{"text":"First input text"}]},{"role":"user","parts":[{"text":"Second input text"}]}]', + ); + }, + }) .start() .completed(); }); @@ -656,22 +439,6 @@ describe('Google GenAI integration', () => { const longContent = 'A'.repeat(50_000); - const EXPECTED_TRANSACTION_NO_TRUNCATION = { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([ - { role: 'user', parts: [{ text: longContent }] }, - { role: 'model', parts: [{ text: 'Some reply' }] }, - { role: 'user', parts: [{ text: 'Follow-up question' }] }, - ]), - [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3, - }), - }), - ]), - }; - createEsmAndCjsTests( __dirname, 'scenario-no-truncation.mjs', @@ -680,7 +447,23 @@ describe('Google GenAI integration', () => { test('does not truncate input messages when enableTruncation is false', async () => { await createRunner() .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_NO_TRUNCATION }) + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(1); + const [firstSpan] = container.items; + + // [0] generate_content with full (non-truncated) input messages + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( + JSON.stringify([ + { role: 'user', parts: [{ text: longContent }] }, + { role: 'model', parts: [{ text: 'Some reply' }] }, + { role: 'user', parts: [{ text: 'Follow-up question' }] }, + ]), + ); + expect(firstSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE].value).toBe(3); + }, + }) .start() .completed(); }); @@ -694,12 +477,41 @@ describe('Google GenAI integration', () => { await createRunner() .expect({ span: container => { - const spans = container.items; + expect(container.items).toHaveLength(8); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan, eighthSpan] = + container.items; - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), - ); - expect(chatSpan).toBeDefined(); + // [0] express middleware: query + expect(firstSpan!.name).toBe('query'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [1] express middleware: expressInit + expect(secondSpan!.name).toBe('expressInit'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [2] express middleware: jsonParser + expect(thirdSpan!.name).toBe('jsonParser'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [3] express request handler + expect(fourthSpan!.name).toBe('/v1beta/models/:model\\:generateContent'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('request_handler.express'); + + // [4] http.server span for the mock endpoint + expect(fifthSpan!.name).toBe('POST /v1beta/models/gemini-1.5-flash:generateContent'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('http.server'); + + // [5] outgoing POST client span + expect(sixthSpan!.name).toBe('POST'); + + // [6] google-genai generate_content span with full (non-truncated) input + expect(seventhSpan!.name).toBe('generate_content gemini-1.5-flash'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toContain(streamingLongContent); + + // [7] main transaction span + expect(eighthSpan!.name).toBe('main'); + expect(eighthSpan!.attributes['sentry.op'].value).toBe('function'); }, }) .start() @@ -716,19 +528,46 @@ describe('Google GenAI integration', () => { await createRunner() .expect({ span: container => { - const spans = container.items; - - // With explicit enableTruncation: true, content should be truncated despite streaming. - // Find the chat span by matching the start of the truncated content (the 'A' repeated messages). - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith( - '[{"role":"user","parts":[{"text":"AAAA', - ), + expect(container.items).toHaveLength(8); + const [firstSpan, secondSpan, thirdSpan, fourthSpan, fifthSpan, sixthSpan, seventhSpan, eighthSpan] = + container.items; + + // [0] express middleware: query + expect(firstSpan!.name).toBe('query'); + expect(firstSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [1] express middleware: expressInit + expect(secondSpan!.name).toBe('expressInit'); + expect(secondSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [2] express middleware: jsonParser + expect(thirdSpan!.name).toBe('jsonParser'); + expect(thirdSpan!.attributes['sentry.op'].value).toBe('middleware.express'); + + // [3] express request handler + expect(fourthSpan!.name).toBe('/v1beta/models/:model\\:generateContent'); + expect(fourthSpan!.attributes['sentry.op'].value).toBe('request_handler.express'); + + // [4] http.server span for the mock endpoint + expect(fifthSpan!.name).toBe('POST /v1beta/models/gemini-1.5-flash:generateContent'); + expect(fifthSpan!.attributes['sentry.op'].value).toBe('http.server'); + + // [5] outgoing POST client span + expect(sixthSpan!.name).toBe('POST'); + + // [6] google-genai generate_content with explicitly truncated input despite streaming + expect(seventhSpan!.name).toBe('generate_content gemini-1.5-flash'); + expect(seventhSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toMatch( + /^\[\{"role":"user","parts":\[\{"text":"AAAA/, ); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( + expect(seventhSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( streamingLongContent.length, ); + + // [7] main transaction span + expect(eighthSpan!.name).toBe('main'); + expect(eighthSpan!.attributes['sentry.op'].value).toBe('function'); }, }) .start() From 26f9157aa0540bc839c39b935586782be1e91344 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 12:38:44 +0900 Subject: [PATCH 17/18] Update Google GenAI Cloudflare tests Co-Authored-By: Claude Opus 4.6 --- .../suites/tracing/google-genai/test.ts | 113 +++++++++--------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/test.ts index 5194e3d3a581..98f730af9716 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/test.ts @@ -1,4 +1,3 @@ -import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; import { expect, it } from 'vitest'; import { GEN_AI_OPERATION_NAME_ATTRIBUTE, @@ -18,67 +17,69 @@ import { createRunner } from '../../../runner'; // want to test that the instrumentation does not break in our // cloudflare SDK. -it('traces Google GenAI chat creation and message sending', async () => { +it('traces Google GenAI chat creation and message sending', async ({ signal }) => { const runner = createRunner(__dirname) .ignore('event') .expect(envelope => { + // Transaction item (first item in envelope) const transactionEvent = envelope[1]?.[0]?.[1] as any; - expect(transactionEvent.transaction).toBe('GET /'); - expect(transactionEvent.spans).toEqual( - expect.arrayContaining([ - // chat.sendMessage - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-pro', - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }), - description: 'chat gemini-1.5-pro', - op: 'gen_ai.chat', - origin: 'auto.ai.google_genai', - }), - // models.generateContent - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-flash', - [GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7, - [GEN_AI_REQUEST_TOP_P_ATTRIBUTE]: 0.9, - [GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100, - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8, - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12, - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20, - }), - description: 'generate_content gemini-1.5-flash', - op: 'gen_ai.generate_content', - origin: 'auto.ai.google_genai', - }), - // models.embedContent - expect.objectContaining({ - data: expect.objectContaining({ - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.embeddings', - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai', - [GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'text-embedding-004', - }), - description: 'embeddings text-embedding-004', - op: 'gen_ai.embeddings', - origin: 'auto.ai.google_genai', - }), - ]), - ); + + // Span container item (second item in same envelope) + const container = envelope[1]?.[1]?.[1] as any; + expect(container).toBeDefined(); + expect(container.items).toHaveLength(3); + const [firstSpan, secondSpan, thirdSpan] = container.items; + + // [0] chat gemini-1.5-pro + expect(firstSpan!.name).toBe('chat gemini-1.5-pro'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'chat' }); + expect(firstSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(firstSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.google_genai' }); + expect(firstSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'google_genai' }); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gemini-1.5-pro', + }); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 12 }); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 }); + + // [1] generate_content gemini-1.5-flash + expect(secondSpan!.name).toBe('generate_content gemini-1.5-flash'); + expect(secondSpan!.status).toBe('ok'); + expect(secondSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(secondSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.generate_content' }); + expect(secondSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.google_genai' }); + expect(secondSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'google_genai' }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gemini-1.5-flash', + }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]).toEqual({ type: 'double', value: 0.7 }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE]).toEqual({ type: 'double', value: 0.9 }); + expect(secondSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 100 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 8 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 12 }); + expect(secondSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ type: 'integer', value: 20 }); + + // [2] embeddings text-embedding-004 + expect(thirdSpan!.name).toBe('embeddings text-embedding-004'); + expect(thirdSpan!.status).toBe('ok'); + expect(thirdSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ type: 'string', value: 'embeddings' }); + expect(thirdSpan!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.embeddings' }); + expect(thirdSpan!.attributes['sentry.origin']).toEqual({ type: 'string', value: 'auto.ai.google_genai' }); + expect(thirdSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'google_genai' }); + expect(thirdSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'text-embedding-004', + }); }) - .start(); + .start(signal); await runner.makeRequest('get', '/'); await runner.completed(); }); From 76d835cc926e75931d6466a299fc3255972a2d7a Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 20 Apr 2026 14:35:30 +0900 Subject: [PATCH 18/18] Update size limits Co-Authored-By: Claude Opus 4.6 --- .size-limit.js | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.size-limit.js b/.size-limit.js index 718781bbd318..46a396ad882d 100644 --- a/.size-limit.js +++ b/.size-limit.js @@ -8,7 +8,7 @@ module.exports = [ path: 'packages/browser/build/npm/esm/prod/index.js', import: createImport('init'), gzip: true, - limit: '26 KB', + limit: '27 KB', }, { name: '@sentry/browser - with treeshaking flags', @@ -38,7 +38,7 @@ module.exports = [ path: 'packages/browser/build/npm/esm/prod/index.js', import: createImport('init', 'browserTracingIntegration'), gzip: true, - limit: '44 KB', + limit: '45 KB', }, { name: '@sentry/browser (incl. Tracing + Span Streaming)', @@ -52,14 +52,14 @@ module.exports = [ path: 'packages/browser/build/npm/esm/prod/index.js', import: createImport('init', 'browserTracingIntegration', 'browserProfilingIntegration'), gzip: true, - limit: '49 KB', + limit: '50 KB', }, { name: '@sentry/browser (incl. Tracing, Replay)', path: 'packages/browser/build/npm/esm/prod/index.js', import: createImport('init', 'browserTracingIntegration', 'replayIntegration'), gzip: true, - limit: '83 KB', + limit: '84 KB', }, { name: '@sentry/browser (incl. Tracing, Replay) - with treeshaking flags', @@ -89,21 +89,21 @@ module.exports = [ path: 'packages/browser/build/npm/esm/prod/index.js', import: createImport('init', 'browserTracingIntegration', 'replayIntegration', 'replayCanvasIntegration'), gzip: true, - limit: '88 KB', + limit: '89 KB', }, { name: '@sentry/browser (incl. Tracing, Replay, Feedback)', path: 'packages/browser/build/npm/esm/prod/index.js', import: createImport('init', 'browserTracingIntegration', 'replayIntegration', 'feedbackIntegration'), gzip: true, - limit: '100 KB', + limit: '101 KB', }, { name: '@sentry/browser (incl. Feedback)', path: 'packages/browser/build/npm/esm/prod/index.js', import: createImport('init', 'feedbackIntegration'), gzip: true, - limit: '43 KB', + limit: '44 KB', }, { name: '@sentry/browser (incl. sendFeedback)', @@ -117,7 +117,7 @@ module.exports = [ path: 'packages/browser/build/npm/esm/prod/index.js', import: createImport('init', 'feedbackAsyncIntegration'), gzip: true, - limit: '36 KB', + limit: '37 KB', }, { name: '@sentry/browser (incl. Metrics)', @@ -138,7 +138,7 @@ module.exports = [ path: 'packages/browser/build/npm/esm/prod/index.js', import: createImport('init', 'metrics', 'logger'), gzip: true, - limit: '28 KB', + limit: '29 KB', }, // React SDK (ESM) { @@ -147,7 +147,7 @@ module.exports = [ import: createImport('init', 'ErrorBoundary'), ignore: ['react/jsx-runtime'], gzip: true, - limit: '28 KB', + limit: '29 KB', }, { name: '@sentry/react (incl. Tracing)', @@ -163,14 +163,14 @@ module.exports = [ path: 'packages/vue/build/esm/index.js', import: createImport('init'), gzip: true, - limit: '31 KB', + limit: '32 KB', }, { name: '@sentry/vue (incl. Tracing)', path: 'packages/vue/build/esm/index.js', import: createImport('init', 'browserTracingIntegration'), gzip: true, - limit: '46 KB', + limit: '47 KB', }, // Svelte SDK (ESM) { @@ -178,7 +178,7 @@ module.exports = [ path: 'packages/svelte/build/esm/index.js', import: createImport('init'), gzip: true, - limit: '26 KB', + limit: '27 KB', }, // Browser CDN bundles { @@ -191,25 +191,25 @@ module.exports = [ name: 'CDN Bundle (incl. Tracing)', path: createCDNPath('bundle.tracing.min.js'), gzip: true, - limit: '45 KB', + limit: '46 KB', }, { name: 'CDN Bundle (incl. Logs, Metrics)', path: createCDNPath('bundle.logs.metrics.min.js'), gzip: true, - limit: '30 KB', + limit: '31 KB', }, { name: 'CDN Bundle (incl. Tracing, Logs, Metrics)', path: createCDNPath('bundle.tracing.logs.metrics.min.js'), gzip: true, - limit: '46 KB', + limit: '47 KB', }, { name: 'CDN Bundle (incl. Replay, Logs, Metrics)', path: createCDNPath('bundle.replay.logs.metrics.min.js'), gzip: true, - limit: '69 KB', + limit: '70 KB', }, { name: 'CDN Bundle (incl. Tracing, Replay)', @@ -221,7 +221,7 @@ module.exports = [ name: 'CDN Bundle (incl. Tracing, Replay, Logs, Metrics)', path: createCDNPath('bundle.tracing.replay.logs.metrics.min.js'), gzip: true, - limit: '83 KB', + limit: '84 KB', }, { name: 'CDN Bundle (incl. Tracing, Replay, Feedback)', @@ -241,7 +241,7 @@ module.exports = [ path: createCDNPath('bundle.min.js'), gzip: false, brotli: false, - limit: '84 KB', + limit: '85 KB', }, { name: 'CDN Bundle (incl. Tracing) - uncompressed', @@ -255,14 +255,14 @@ module.exports = [ path: createCDNPath('bundle.logs.metrics.min.js'), gzip: false, brotli: false, - limit: '88 KB', + limit: '89 KB', }, { name: 'CDN Bundle (incl. Tracing, Logs, Metrics) - uncompressed', path: createCDNPath('bundle.tracing.logs.metrics.min.js'), gzip: false, brotli: false, - limit: '138 KB', + limit: '139 KB', }, { name: 'CDN Bundle (incl. Replay, Logs, Metrics) - uncompressed', @@ -276,28 +276,28 @@ module.exports = [ path: createCDNPath('bundle.tracing.replay.min.js'), gzip: false, brotli: false, - limit: '252 KB', + limit: '253 KB', }, { name: 'CDN Bundle (incl. Tracing, Replay, Logs, Metrics) - uncompressed', path: createCDNPath('bundle.tracing.replay.logs.metrics.min.js'), gzip: false, brotli: false, - limit: '255 KB', + limit: '256 KB', }, { name: 'CDN Bundle (incl. Tracing, Replay, Feedback) - uncompressed', path: createCDNPath('bundle.tracing.replay.feedback.min.js'), gzip: false, brotli: false, - limit: '265 KB', + limit: '266 KB', }, { name: 'CDN Bundle (incl. Tracing, Replay, Feedback, Logs, Metrics) - uncompressed', path: createCDNPath('bundle.tracing.replay.feedback.logs.metrics.min.js'), gzip: false, brotli: false, - limit: '268 KB', + limit: '269 KB', }, // Next.js SDK (ESM) {