From b514411a9abe6b8ca5627cfdb6c5df521e1a1159 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 15 Apr 2026 10:16:28 +0100 Subject: [PATCH 1/6] feat(observability): add config-driven observability selection --- src/boundaries/env-parser.ts | 13 ++++- src/observability/ai-observability.ts | 13 +++++ src/observability/factory.ts | 22 +++++++++ src/observability/langfuse-observability.ts | 21 ++++++++ src/observability/noop-observability.ts | 11 +++++ src/schemas/env-schemas.ts | 35 ++++++++++++- tests/env-parser.test.ts | 35 ++++++++++++- tests/observability/factory.test.ts | 49 +++++++++++++++++++ .../observability/noop-observability.test.ts | 19 +++++++ 9 files changed, 214 insertions(+), 4 deletions(-) create mode 100644 src/observability/ai-observability.ts create mode 100644 src/observability/factory.ts create mode 100644 src/observability/langfuse-observability.ts create mode 100644 src/observability/noop-observability.ts create mode 100644 tests/observability/factory.test.ts create mode 100644 tests/observability/noop-observability.test.ts diff --git a/src/boundaries/env-parser.ts b/src/boundaries/env-parser.ts index c16b270..a31340e 100644 --- a/src/boundaries/env-parser.ts +++ b/src/boundaries/env-parser.ts @@ -28,7 +28,7 @@ function formatProviderValidationError(zodError: z.ZodError, env: unknown): stri ); if (discriminatorIssue) { - return `LLM_PROVIDER is required and must be either 'azure-openai', 'anthropic', or 'openai'. Received: ${providerType ?? 'undefined'}`; + return `LLM_PROVIDER is required and must be one of 'azure-openai', 'anthropic', 'openai', 'gemini', or 'amazon-bedrock'. Received: ${providerType ?? 'undefined'}`; } // Check for missing required fields based on provider type @@ -59,6 +59,17 @@ function formatProviderValidationError(zodError: z.ZodError, env: unknown): stri } } + if (envObj.OBSERVABILITY_BACKEND === 'langfuse') { + const langfuseFields = issues + .filter((issue) => issue.path.length > 0 && String(issue.path[0]).startsWith('LANGFUSE_')) + .map((issue) => issue.path.join('.')); + const missingLangfuseFields = [...new Set(langfuseFields)]; + + if (missingLangfuseFields.length > 0) { + return `Missing required Langfuse observability environment variables: ${missingLangfuseFields.join(', ')}. When using OBSERVABILITY_BACKEND=langfuse, ensure LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and LANGFUSE_BASE_URL are set.`; + } + } + // Check for validation errors (e.g., invalid URL, number out of range) const validationIssues = issues.filter(issue => issue.code === 'invalid_string' || diff --git a/src/observability/ai-observability.ts b/src/observability/ai-observability.ts new file mode 100644 index 0000000..5369c63 --- /dev/null +++ b/src/observability/ai-observability.ts @@ -0,0 +1,13 @@ +export interface AIExecutionContext { + operation: 'structured-eval' | 'agent-tool-loop'; + provider: string; + model: string; + evaluator?: string; + rule?: string; +} + +export interface AIObservability { + init(): Promise | void; + decorateCall(context: AIExecutionContext): Record; + shutdown?(): Promise | void; +} diff --git a/src/observability/factory.ts b/src/observability/factory.ts new file mode 100644 index 0000000..f066994 --- /dev/null +++ b/src/observability/factory.ts @@ -0,0 +1,22 @@ +import type { Logger } from '../logging/logger'; +import type { EnvConfig } from '../schemas/env-schemas'; +import type { AIObservability } from './ai-observability'; +import { LangfuseObservability } from './langfuse-observability'; +import { NoopObservability } from './noop-observability'; + +export function createObservability(env: EnvConfig, logger?: Logger): AIObservability { + if (env.OBSERVABILITY_BACKEND !== 'langfuse') { + return new NoopObservability(); + } + + if (!env.LANGFUSE_PUBLIC_KEY || !env.LANGFUSE_SECRET_KEY || !env.LANGFUSE_BASE_URL) { + throw new Error('Langfuse observability requires LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and LANGFUSE_BASE_URL'); + } + + return new LangfuseObservability({ + publicKey: env.LANGFUSE_PUBLIC_KEY, + secretKey: env.LANGFUSE_SECRET_KEY, + baseUrl: env.LANGFUSE_BASE_URL, + logger, + }); +} diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts new file mode 100644 index 0000000..5035b14 --- /dev/null +++ b/src/observability/langfuse-observability.ts @@ -0,0 +1,21 @@ +import type { Logger } from '../logging/logger'; +import type { AIExecutionContext, AIObservability } from './ai-observability'; + +export interface LangfuseObservabilityConfig { + publicKey: string; + secretKey: string; + baseUrl: string; + logger?: Logger; +} + +export class LangfuseObservability implements AIObservability { + constructor(private readonly _config: LangfuseObservabilityConfig) {} + + init(): void {} + + decorateCall(_context: AIExecutionContext): Record { + return {}; + } + + shutdown(): void {} +} diff --git a/src/observability/noop-observability.ts b/src/observability/noop-observability.ts new file mode 100644 index 0000000..83f0e09 --- /dev/null +++ b/src/observability/noop-observability.ts @@ -0,0 +1,11 @@ +import type { AIExecutionContext, AIObservability } from './ai-observability'; + +export class NoopObservability implements AIObservability { + init(): void {} + + decorateCall(_context: AIExecutionContext): Record { + return {}; + } + + shutdown(): void {} +} diff --git a/src/schemas/env-schemas.ts b/src/schemas/env-schemas.ts index ce2f5a5..fa9cea7 100644 --- a/src/schemas/env-schemas.ts +++ b/src/schemas/env-schemas.ts @@ -63,11 +63,18 @@ const BEDROCK_CONFIG_SCHEMA = z.object({ BEDROCK_TEMPERATURE: z.coerce.number().min(0).max(1).optional(), }); +const OBSERVABILITY_ENV_SCHEMA = z.object({ + OBSERVABILITY_BACKEND: z.enum(['langfuse']).optional(), + LANGFUSE_PUBLIC_KEY: z.string().min(1).optional(), + LANGFUSE_SECRET_KEY: z.string().min(1).optional(), + LANGFUSE_BASE_URL: z.string().url().optional(), +}); + // Base environment schema with shared optional variables const BASE_ENV_SCHEMA = z.object({ INPUT_PRICE_PER_MILLION: z.coerce.number().positive().optional(), OUTPUT_PRICE_PER_MILLION: z.coerce.number().positive().optional(), -}); +}).merge(OBSERVABILITY_ENV_SCHEMA); // Discriminated union based on provider type export const ENV_SCHEMA = z.discriminatedUnion('LLM_PROVIDER', [ @@ -87,6 +94,32 @@ export const ENV_SCHEMA = z.discriminatedUnion('LLM_PROVIDER', [ }); } } + + if (data.OBSERVABILITY_BACKEND === 'langfuse') { + if (!data.LANGFUSE_PUBLIC_KEY) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['LANGFUSE_PUBLIC_KEY'], + message: 'LANGFUSE_PUBLIC_KEY is required when OBSERVABILITY_BACKEND=langfuse', + }); + } + + if (!data.LANGFUSE_SECRET_KEY) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['LANGFUSE_SECRET_KEY'], + message: 'LANGFUSE_SECRET_KEY is required when OBSERVABILITY_BACKEND=langfuse', + }); + } + + if (!data.LANGFUSE_BASE_URL) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['LANGFUSE_BASE_URL'], + message: 'LANGFUSE_BASE_URL is required when OBSERVABILITY_BACKEND=langfuse', + }); + } + } }); export const GLOBAL_CONFIG_SCHEMA = z.object({ diff --git a/tests/env-parser.test.ts b/tests/env-parser.test.ts index fb6b50b..a4faf69 100644 --- a/tests/env-parser.test.ts +++ b/tests/env-parser.test.ts @@ -208,7 +208,7 @@ describe('Environment Parser', () => { }; expect(() => parseEnvironment(env)).toThrow(ValidationError); - expect(() => parseEnvironment(env)).toThrow(/LLM_PROVIDER is required and must be either 'azure-openai', 'anthropic', or 'openai'/); + expect(() => parseEnvironment(env)).toThrow(/LLM_PROVIDER is required and must be one of 'azure-openai', 'anthropic', 'openai', 'gemini', or 'amazon-bedrock'/); }); it('provides specific error message for missing Azure OpenAI variables', () => { @@ -303,4 +303,35 @@ describe('Environment Parser', () => { expect(() => parseEnvironment(env)).toThrow(/Invalid environment variable values/); }); }); -}); \ No newline at end of file + + describe('Observability Configuration', () => { + it('parses Langfuse observability configuration', () => { + const env = { + LLM_PROVIDER: 'openai', + OPENAI_API_KEY: 'sk-test', + OBSERVABILITY_BACKEND: 'langfuse', + LANGFUSE_PUBLIC_KEY: 'pk-lf-test', + LANGFUSE_SECRET_KEY: 'sk-lf-test', + LANGFUSE_BASE_URL: 'https://cloud.langfuse.com', + }; + + const result = parseEnvironment(env); + + expect(result.OBSERVABILITY_BACKEND).toBe('langfuse'); + expect(result.LANGFUSE_PUBLIC_KEY).toBe('pk-lf-test'); + expect(result.LANGFUSE_SECRET_KEY).toBe('sk-lf-test'); + expect(result.LANGFUSE_BASE_URL).toBe('https://cloud.langfuse.com'); + }); + + it('requires Langfuse credentials when OBSERVABILITY_BACKEND is langfuse', () => { + const env = { + LLM_PROVIDER: 'openai', + OPENAI_API_KEY: 'sk-test', + OBSERVABILITY_BACKEND: 'langfuse', + }; + + expect(() => parseEnvironment(env)).toThrow(ValidationError); + expect(() => parseEnvironment(env)).toThrow(/Missing required Langfuse observability environment variables.*LANGFUSE_PUBLIC_KEY.*LANGFUSE_SECRET_KEY.*LANGFUSE_BASE_URL/); + }); + }); +}); diff --git a/tests/observability/factory.test.ts b/tests/observability/factory.test.ts new file mode 100644 index 0000000..5db00ee --- /dev/null +++ b/tests/observability/factory.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest'; +import { ProviderType } from '../../src/providers/provider-factory'; +import type { EnvConfig } from '../../src/schemas/env-schemas'; +import { createObservability } from '../../src/observability/factory'; +import { LangfuseObservability } from '../../src/observability/langfuse-observability'; +import { NoopObservability } from '../../src/observability/noop-observability'; + +describe('createObservability', () => { + const baseEnv: EnvConfig = { + LLM_PROVIDER: ProviderType.OpenAI, + OPENAI_API_KEY: 'sk-test-key', + OPENAI_MODEL: 'gpt-4o', + }; + + it('returns NoopObservability when no backend is configured', () => { + expect(createObservability(baseEnv)).toBeInstanceOf(NoopObservability); + }); + + it('returns NoopObservability when backend is not langfuse', () => { + const env = { + ...baseEnv, + OBSERVABILITY_BACKEND: undefined, + }; + + expect(createObservability(env)).toBeInstanceOf(NoopObservability); + }); + + it('returns LangfuseObservability when OBSERVABILITY_BACKEND is langfuse', () => { + const env: EnvConfig = { + ...baseEnv, + OBSERVABILITY_BACKEND: 'langfuse', + LANGFUSE_PUBLIC_KEY: 'pk-lf-test', + LANGFUSE_SECRET_KEY: 'sk-lf-test', + LANGFUSE_BASE_URL: 'https://cloud.langfuse.com', + }; + + expect(createObservability(env)).toBeInstanceOf(LangfuseObservability); + }); + + it('throws when langfuse backend is selected without required keys', () => { + const env = { + ...baseEnv, + OBSERVABILITY_BACKEND: 'langfuse', + LANGFUSE_PUBLIC_KEY: 'pk-lf-test', + } as EnvConfig; + + expect(() => createObservability(env)).toThrow(/Langfuse observability requires/); + }); +}); diff --git a/tests/observability/noop-observability.test.ts b/tests/observability/noop-observability.test.ts new file mode 100644 index 0000000..877b00f --- /dev/null +++ b/tests/observability/noop-observability.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { NoopObservability } from '../../src/observability/noop-observability'; + +describe('NoopObservability', () => { + const subject = new NoopObservability(); + + it('returns an empty option object for any AI execution context', () => { + expect(subject.decorateCall({ + operation: 'structured-eval', + provider: 'openai', + model: 'gpt-4o', + })).toEqual({}); + }); + + it('allows init and shutdown to be called without throwing', async () => { + expect(() => subject.init()).not.toThrow(); + await expect(Promise.resolve(subject.shutdown?.())).resolves.toBeUndefined(); + }); +}); From f6a262c061d93f91ed29391497f3a110fa013e17 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 15 Apr 2026 10:21:20 +0100 Subject: [PATCH 2/6] feat(providers): inject observability into AI calls --- src/providers/provider-factory.ts | 3 + src/providers/vercel-ai-provider.ts | 73 ++++++++++++++++ tests/provider-factory.test.ts | 22 +++++ .../vercel-ai-provider-agent-loop.test.ts | 85 +++++++++++++++++++ tests/vercel-ai-provider.test.ts | 79 ++++++++++++++++- 5 files changed, 261 insertions(+), 1 deletion(-) diff --git a/src/providers/provider-factory.ts b/src/providers/provider-factory.ts index d40a02f..83d81b3 100644 --- a/src/providers/provider-factory.ts +++ b/src/providers/provider-factory.ts @@ -9,12 +9,14 @@ import { VercelAIProvider, type VercelAIConfig } from './vercel-ai-provider'; import { RequestBuilder } from './request-builder'; import type { EnvConfig } from '../schemas/env-schemas'; import type { Logger } from '../logging/logger'; +import type { AIObservability } from '../observability/ai-observability'; export interface ProviderOptions { debug?: boolean; showPrompt?: boolean; showPromptTrunc?: boolean; logger?: Logger; + observability?: AIObservability; } export enum ProviderType { @@ -106,6 +108,7 @@ export function createProvider( ...(options.showPrompt !== undefined && { showPrompt: options.showPrompt }), ...(options.showPromptTrunc !== undefined && { showPromptTrunc: options.showPromptTrunc }), ...(options.logger ? { logger: options.logger } : {}), + ...(options.observability ? { observability: options.observability } : {}), }; return new VercelAIProvider(config, builder); diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 81326f0..dad0827 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -5,6 +5,7 @@ import pLimit from 'p-limit'; import { AgentToolLoopParams, AgentToolLoopResult, LLMProvider, LLMResult } from './llm-provider'; import { DefaultRequestBuilder, RequestBuilder } from './request-builder'; import { createNoopLogger, type Logger } from '../logging/logger'; +import type { AIExecutionContext, AIObservability } from '../observability/ai-observability'; export interface VercelAIConfig { model: LanguageModel; @@ -14,12 +15,14 @@ export interface VercelAIConfig { showPrompt?: boolean; showPromptTrunc?: boolean; logger?: Logger; + observability?: AIObservability; } export class VercelAIProvider implements LLMProvider { private config: VercelAIConfig; private builder: RequestBuilder; private logger: Logger; + private observability?: AIObservability; constructor(config: VercelAIConfig, builder?: RequestBuilder) { this.config = { @@ -29,6 +32,7 @@ export class VercelAIProvider implements LLMProvider { }; this.builder = builder ?? new DefaultRequestBuilder(); this.logger = config.logger ?? createNoopLogger(); + this.observability = config.observability; }; async runPromptStructured( @@ -66,12 +70,21 @@ export class VercelAIProvider implements LLMProvider { } try { + const observabilityOptions = this.getObservabilityOptions({ + operation: 'structured-eval', + provider: this.resolveProviderName(), + model: this.resolveModelName(), + evaluator: this.extractContextValue(context, 'evaluatorName', 'evaluator'), + rule: this.extractContextValue(context, 'ruleName', 'rule'), + }); + const result = await generateText({ model: this.config.model, system: systemPrompt, prompt: `Input:\n\n${content}`, ...(this.config.temperature !== undefined && { temperature: this.config.temperature }), ...(this.config.maxTokens !== undefined && { maxTokens: this.config.maxTokens }), + ...observabilityOptions, output: Output.object({ schema: zodSchema, }), @@ -137,6 +150,11 @@ export class VercelAIProvider implements LLMProvider { system: params.systemPrompt, prompt: params.prompt, ...(params.maxRetries !== undefined ? { maxRetries: params.maxRetries } : {}), + ...this.getObservabilityOptions({ + operation: 'agent-tool-loop', + provider: this.resolveProviderName(), + model: this.resolveModelName(), + }), stopWhen: stepCountIs(params.maxSteps ?? 1000), providerOptions: { openai: { @@ -176,6 +194,61 @@ export class VercelAIProvider implements LLMProvider { }; } + private getObservabilityOptions(context: AIExecutionContext): Record { + if (!this.observability) { + return {}; + } + + try { + return this.observability.decorateCall(context); + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + this.logger.warn('[vectorlint] Failed to decorate AI call for observability; continuing without telemetry options', { + error: err.message, + operation: context.operation, + }); + return {}; + } + } + + private resolveProviderName(): string { + const model = this.config.model as unknown as Record; + const provider = model.provider; + if (typeof provider === 'string' && provider.length > 0) { + return provider; + } + return 'unknown'; + } + + private resolveModelName(): string { + const model = this.config.model as unknown as Record; + for (const key of ['modelId', 'model', 'id']) { + const value = model[key]; + if (typeof value === 'string' && value.length > 0) { + return value; + } + } + return 'unknown'; + } + + private extractContextValue( + context: import('./request-builder').EvalContext | undefined, + ...keys: string[] + ): string | undefined { + if (!context) { + return undefined; + } + + const contextRecord = context as unknown as Record; + for (const key of keys) { + const value = contextRecord[key]; + if (typeof value === 'string' && value.length > 0) { + return value; + } + } + return undefined; + } + /** * Entry point: converts the VectorLint schema wrapper to a Zod schema. * Schema format: { name: string, schema: { type, properties, required, ... } } diff --git a/tests/provider-factory.test.ts b/tests/provider-factory.test.ts index 133bd92..328c861 100644 --- a/tests/provider-factory.test.ts +++ b/tests/provider-factory.test.ts @@ -3,6 +3,7 @@ import { createProvider, ProviderType } from '../src/providers/provider-factory' import { VercelAIProvider } from '../src/providers/vercel-ai-provider'; import { DefaultRequestBuilder } from '../src/providers/request-builder'; import type { EnvConfig } from '../src/schemas/env-schemas'; +import type { AIObservability } from '../src/observability/ai-observability'; // Mock the Vercel AI SDK provider creators vi.mock('@ai-sdk/openai', () => ({ @@ -332,6 +333,27 @@ describe('Provider Factory', () => { }); }); + describe('Observability Wiring', () => { + it('passes observability through to VercelAIProvider', () => { + const envConfig: EnvConfig = { + LLM_PROVIDER: ProviderType.OpenAI, + OPENAI_API_KEY: 'sk-test-key', + OPENAI_MODEL: 'gpt-4o', + }; + const observability: AIObservability = { + init: vi.fn(), + decorateCall: vi.fn(() => ({})), + shutdown: vi.fn(), + }; + + const provider = createProvider(envConfig, { observability }) as unknown as { + config?: { observability?: AIObservability }; + }; + + expect(provider.config?.observability).toBe(observability); + }); + }); + describe('Provider-Specific Configuration', () => { it('handles Azure OpenAI specific fields correctly', () => { const envConfig: EnvConfig = { diff --git a/tests/providers/vercel-ai-provider-agent-loop.test.ts b/tests/providers/vercel-ai-provider-agent-loop.test.ts index eb4e83d..994aa36 100644 --- a/tests/providers/vercel-ai-provider-agent-loop.test.ts +++ b/tests/providers/vercel-ai-provider-agent-loop.test.ts @@ -97,6 +97,91 @@ describe('VercelAIProvider agent loop', () => { }); }); + it('adds observability options to agent-loop generateText calls', async () => { + const observability = { + init: vi.fn(), + decorateCall: vi.fn(() => ({ + experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.agent-tool-loop' }, + })), + shutdown: vi.fn(), + }; + + MOCK_GENERATE_TEXT.mockResolvedValue({ + text: 'done', + usage: { inputTokens: 10, outputTokens: 5 }, + steps: [], + finishReason: 'stop', + }); + + const provider = new VercelAIProvider({ + model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, + observability, + }); + + await provider.runAgentToolLoop({ + systemPrompt: 'system', + prompt: 'prompt', + tools: { + finalize_review: { + description: 'Finalize review session', + inputSchema: z.object({ summary: z.string().optional() }), + execute: () => Promise.resolve({ ok: true }), + }, + }, + }); + + expect(observability.decorateCall).toHaveBeenCalledWith({ + operation: 'agent-tool-loop', + provider: 'openai', + model: 'gpt-4o-mini', + }); + const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; + expect(call).toHaveProperty('experimental_telemetry'); + }); + + it('continues agent-loop AI calls when observability decoration fails', async () => { + const logger = createMockLogger(); + const observability = { + init: vi.fn(), + decorateCall: vi.fn(() => { + throw new Error('telemetry failed'); + }), + shutdown: vi.fn(), + }; + + MOCK_GENERATE_TEXT.mockResolvedValue({ + text: 'done', + usage: { inputTokens: 10, outputTokens: 5 }, + steps: [], + finishReason: 'stop', + }); + + const provider = new VercelAIProvider({ + model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, + logger, + observability, + }); + + await provider.runAgentToolLoop({ + systemPrompt: 'system', + prompt: 'prompt', + tools: { + finalize_review: { + description: 'Finalize review session', + inputSchema: z.object({ summary: z.string().optional() }), + execute: () => Promise.resolve({ ok: true }), + }, + }, + }); + + const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; + expect(call).not.toHaveProperty('experimental_telemetry'); + expect(logger.warn).toHaveBeenCalledWith( + '[vectorlint] Failed to decorate AI call for observability; continuing without telemetry options', + expect.objectContaining({ error: 'telemetry failed', operation: 'agent-tool-loop' }) + ); + }); + it('limits concurrent tool executes to maxParallelToolCalls', async () => { let maxConcurrent = 0; let currentConcurrent = 0; diff --git a/tests/vercel-ai-provider.test.ts b/tests/vercel-ai-provider.test.ts index 45b76b2..605b687 100644 --- a/tests/vercel-ai-provider.test.ts +++ b/tests/vercel-ai-provider.test.ts @@ -44,7 +44,10 @@ import { createMockLogger } from './utils'; // Mock model stub — only stored in config and passed through to the mocked // generateText function, so it doesn't need to implement the full interface. -const MOCK_MODEL = {} as unknown as LanguageModel; +const MOCK_MODEL = { + provider: 'openai', + modelId: 'gpt-4o', +} as unknown as LanguageModel; describe('VercelAIProvider', () => { beforeEach(() => { @@ -218,6 +221,80 @@ describe('VercelAIProvider', () => { }) ); }); + + it('adds observability options to structured generateText calls', async () => { + const observability = { + init: vi.fn(), + decorateCall: vi.fn(() => ({ + experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.structured-eval' }, + })), + shutdown: vi.fn(), + }; + + MOCK_GENERATE_TEXT.mockResolvedValue({ output: { result: 'success' } }); + + const provider = new VercelAIProvider({ + model: MOCK_MODEL, + observability, + }); + + await provider.runPromptStructured( + 'Test content', + 'Test prompt', + { + name: 'test_schema', + schema: { properties: { result: { type: 'string' } }, type: 'object' }, + } + ); + + expect(observability.decorateCall).toHaveBeenCalledWith({ + operation: 'structured-eval', + provider: 'openai', + model: 'gpt-4o', + evaluator: undefined, + rule: undefined, + }); + expect(MOCK_GENERATE_TEXT).toHaveBeenCalledWith( + expect.objectContaining({ + experimental_telemetry: { isEnabled: true, functionId: 'vectorlint.structured-eval' }, + }) + ); + }); + + it('continues structured AI calls when observability decoration fails', async () => { + const logger = createMockLogger(); + const observability = { + init: vi.fn(), + decorateCall: vi.fn(() => { + throw new Error('telemetry failed'); + }), + shutdown: vi.fn(), + }; + + MOCK_GENERATE_TEXT.mockResolvedValue({ output: { result: 'success' } }); + + const provider = new VercelAIProvider({ + model: MOCK_MODEL, + logger, + observability, + }); + + await provider.runPromptStructured( + 'Test content', + 'Test prompt', + { + name: 'test_schema', + schema: { properties: { result: { type: 'string' } }, type: 'object' }, + } + ); + + const call = MOCK_GENERATE_TEXT.mock.calls.at(-1)?.[0] as Record; + expect(call).not.toHaveProperty('experimental_telemetry'); + expect(logger.warn).toHaveBeenCalledWith( + '[vectorlint] Failed to decorate AI call for observability; continuing without telemetry options', + expect.objectContaining({ error: 'telemetry failed', operation: 'structured-eval' }) + ); + }); }); describe('Error Handling', () => { From 735ecc88d943e6a0230f8b2cb5274671f38bc3c7 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 15 Apr 2026 10:54:44 +0100 Subject: [PATCH 3/6] feat(observability): add Langfuse runtime integration --- package-lock.json | 950 +++++++++++++++++- package.json | 6 +- src/boundaries/env-parser.ts | 2 +- src/cli/commands.ts | 140 ++- src/observability/factory.ts | 6 +- src/observability/langfuse-observability.ts | 82 +- src/observability/noop-observability.ts | 3 +- src/schemas/env-schemas.ts | 8 - tests/env-parser.test.ts | 2 +- tests/main-command-observability.test.ts | 191 ++++ tests/observability/factory.test.ts | 11 + .../langfuse-observability.test.ts | 127 +++ 12 files changed, 1441 insertions(+), 87 deletions(-) create mode 100644 tests/main-command-observability.test.ts create mode 100644 tests/observability/langfuse-observability.test.ts diff --git a/package-lock.json b/package-lock.json index aebca73..3fc9064 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,12 +15,16 @@ "@ai-sdk/google": "^3.0.31", "@ai-sdk/openai": "^3.0.33", "@ai-sdk/perplexity": "^1.0.0", + "@langfuse/otel": "^5.1.0", + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/sdk-node": "^0.214.0", "@types/micromatch": "^4.0.9", "ai": "^6.0.99", "chalk": "^5.3.0", "commander": "^12.0.0", "fast-glob": "^3.3.2", "fuzzball": "^2.2.3", + "langfuse": "^3.38.20", "micromatch": "^4.0.5", "ora": "^8.2.0", "p-limit": "^7.3.0", @@ -1176,6 +1180,37 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", + "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz", + "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.3", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -1318,6 +1353,43 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@langfuse/core": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@langfuse/core/-/core-5.1.0.tgz", + "integrity": "sha512-yFvC67HBtrY4B3tyzF8+RJaIqK79LBVXtAgtmEc2vhpKauecvSW0zevRnRynFX+ajUHqi9TN7tnD91FJszFLgQ==", + "license": "MIT", + "peerDependencies": { + "@opentelemetry/api": "^1.9.0" + } + }, + "node_modules/@langfuse/otel": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@langfuse/otel/-/otel-5.1.0.tgz", + "integrity": "sha512-pvaXgZHMHqjsRjn+Gs5amrrq61w0Rxz1OChmLr2FfQzlymNl7+MxSXsWBj5dZQlufGbhyG+LT3wdx3MV8aLXHQ==", + "license": "MIT", + "dependencies": { + "@langfuse/core": "^5.1.0" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^2.0.1", + "@opentelemetry/exporter-trace-otlp-http": ">=0.202.0 <1.0.0", + "@opentelemetry/sdk-trace-base": "^2.0.1" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.12", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", @@ -1367,14 +1439,519 @@ } }, "node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", "license": "Apache-2.0", "engines": { "node": ">=8.0.0" } }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.214.0.tgz", + "integrity": "sha512-40lSJeqYO8Uz2Yj7u94/SJWE/wONa7rmMKjI1ZcIjgf3MHNHv1OZUCrCETGuaRF62d5pQD1wKIW+L4lmSMTzZA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/configuration": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/configuration/-/configuration-0.214.0.tgz", + "integrity": "sha512-Q+awuEwxhETwIAXuxHvIY5ZMEP0ZqvxLTi9kclrkyVJppEUXYL3Bhiw3jYrxdHYMh0Y0tVInQH9FEZ1aMinvLA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "yaml": "^2.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.6.1.tgz", + "integrity": "sha512-XHzhwRNkBpeP8Fs/qjGrAf9r9PRv67wkJQ/7ZPaBQQ68DYlTBBx5MF9LvPx7mhuXcDessKK2b+DcxqwpgkcivQ==", + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.6.1.tgz", + "integrity": "sha512-8xHSGWpJP9wBxgBpnqGL0R3PbdWQndL1Qp50qrg71+B28zK5OQmUgcDKLJgzyAAV38t4tOyLMGDD60LneR5W8g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-grpc": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.214.0.tgz", + "integrity": "sha512-SwmFRwO8mi6nndzbsjPgSFg7qy1WeNHRFD+s6uCsdiUDUt3+yzI2qiHE3/ub2f37+/CbeGcG+Ugc8Gwr6nu2Aw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.14.3", + "@opentelemetry/core": "2.6.1", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.214.0", + "@opentelemetry/otlp-transformer": "0.214.0", + "@opentelemetry/sdk-logs": "0.214.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-http": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.214.0.tgz", + "integrity": "sha512-9qv2Tl/Hq6qc5pJCbzFJnzA0uvlb9DgM70yGJPYf3bA5LlLkRCpcn81i4JbcIH4grlQIWY6A+W7YG0LLvS1BAw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.214.0", + "@opentelemetry/core": "2.6.1", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/otlp-transformer": "0.214.0", + "@opentelemetry/sdk-logs": "0.214.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-proto/-/exporter-logs-otlp-proto-0.214.0.tgz", + "integrity": "sha512-IWAVvCO1TlpotRjFmhQFz9RSfQy5BsLtDRBtptSrXZRwfyRPpuql/RMe5zdmu0Gxl3ERDFwOzOqkf3bwy7Jzcw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.214.0", + "@opentelemetry/core": "2.6.1", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/otlp-transformer": "0.214.0", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-logs": "0.214.0", + "@opentelemetry/sdk-trace-base": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-grpc": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-grpc/-/exporter-metrics-otlp-grpc-0.214.0.tgz", + "integrity": "sha512-0NGxWHVYHgbp51SEzmsP+Hdups81eRs229STcSWHo3WO0aqY6RpJ9csxfyEtFgaNrBDv6UfOh0je4ss/ROS6XA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.14.3", + "@opentelemetry/core": "2.6.1", + "@opentelemetry/exporter-metrics-otlp-http": "0.214.0", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.214.0", + "@opentelemetry/otlp-transformer": "0.214.0", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-metrics": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.214.0.tgz", + "integrity": "sha512-Tx/59RmjBgkXJ3qnsD04rpDrVWL53LU/czpgLJh+Ab98nAroe91I7vZ3uGN9mxwPS0jsZEnmqmHygVwB2vRMlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/otlp-transformer": "0.214.0", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-metrics": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-proto": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-proto/-/exporter-metrics-otlp-proto-0.214.0.tgz", + "integrity": "sha512-pJIcghFGhx3VSCgP5U+yZx+OMNj0t+ttnhC8IjL5Wza7vWIczctF6t3AGcVQffi2dEqX+ZHANoBwoPR8y6RMKA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/exporter-metrics-otlp-http": "0.214.0", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/otlp-transformer": "0.214.0", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-metrics": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-prometheus": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.214.0.tgz", + "integrity": "sha512-4TGYoZKebUWVuYkV6r5wS2dUF4zH7EbWFw/Uqz1ZM1tGHQeFT9wzHGXq3iSIXMUrwu5jRdxjfMaXrYejPu2kpQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-metrics": "2.6.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.214.0.tgz", + "integrity": "sha512-FWRZ7AWoTryYhthralHkfXUuyO3l7cRsnr49WcDio1orl2a7KxT8aDZdwQtV1adzoUvZ9Gfo+IstElghCS4zfw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.14.3", + "@opentelemetry/core": "2.6.1", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.214.0", + "@opentelemetry/otlp-transformer": "0.214.0", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-trace-base": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.214.0.tgz", + "integrity": "sha512-kIN8nTBMgV2hXzV/a20BCFilPZdAIMYYJGSgfMMRm/Xa+07y5hRDS2Vm12A/z8Cdu3Sq++ZvJfElokX2rkgGgw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/otlp-transformer": "0.214.0", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-trace-base": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-proto": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.214.0.tgz", + "integrity": "sha512-ON0spYWb2yAdQ9b+ItNyK0c6qdtcs+0eVR4YFJkhJL7agfT8sHFg0e5EesauSRiTHPZHiDobI92k77q0lwAmqg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/otlp-transformer": "0.214.0", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-trace-base": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-zipkin": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-zipkin/-/exporter-zipkin-2.6.1.tgz", + "integrity": "sha512-km2/hD3inLTqtLnUAHDGz7ZP/VOyZNslrC/iN66x4jkmpckwlONW54LRPNI6fm09/musDtZga9EWsxgwnjGUlw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-trace-base": "2.6.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/instrumentation": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.214.0.tgz", + "integrity": "sha512-MHqEX5Dk59cqVah5LiARMACku7jXSVk9iVDWOea4x3cr7VfdByeDCURK6o1lntT1JS/Tsovw01UJrBhN3/uC5w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.214.0", + "import-in-the-middle": "^3.0.0", + "require-in-the-middle": "^8.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.214.0.tgz", + "integrity": "sha512-u1Gdv0/E9wP+apqWf7Wv2npXmgJtxsW2XL0TEv9FZloTZRuMBKmu8cYVXwS4Hm3q/f/3FuCnPTgiwYvIqRSpRg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/otlp-transformer": "0.214.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-grpc-exporter-base": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.214.0.tgz", + "integrity": "sha512-IDP6zcyA24RhNZ289MP6eToIZcinlmirHjX8v3zKCQ2ZhPpt5cGwkN91tCth337lqHIgWcTy90uKRiX/SzALDw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.14.3", + "@opentelemetry/core": "2.6.1", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/otlp-transformer": "0.214.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.214.0.tgz", + "integrity": "sha512-DSaYcuBRh6uozfsWN3R8HsN0yDhCuWP7tOFdkUOVaWD1KVJg8m4qiLUsg/tNhTLS9HUYUcwNpwL2eroLtsZZ/w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.214.0", + "@opentelemetry/core": "2.6.1", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-logs": "0.214.0", + "@opentelemetry/sdk-metrics": "2.6.1", + "@opentelemetry/sdk-trace-base": "2.6.1", + "protobufjs": "^7.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/propagator-b3": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-b3/-/propagator-b3-2.6.1.tgz", + "integrity": "sha512-Dvz9TA6cPqIbxolSzQ5x9br6iQlqdGhVYrm+oYc7pfJ7LaVXz8F0XIqhWbnKB5YvfZ6SUmabBUUxnvHs/9uhxA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-jaeger": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-jaeger/-/propagator-jaeger-2.6.1.tgz", + "integrity": "sha512-kKFMxBcjBZAC1vBch5mtZ/dJQvcAEKWga+c+q5iGgRLPIE6Mc649zEwMaCIQCzalziMJQiyUadFYMHmELB7AFw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.6.1.tgz", + "integrity": "sha512-lID/vxSuKWXM55XhAKNoYXu9Cutoq5hFdkbTdI/zDKQktXzcWBVhNsOkiZFTMU9UtEWuGRNe0HUgmsFldIdxVA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.214.0.tgz", + "integrity": "sha512-zf6acnScjhsaBUU22zXZ/sLWim1dfhUAbGXdMmHmNG3LfBnQ3DKsOCITb2IZwoUsNNMTogqFKBnlIPPftUgGwA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.214.0", + "@opentelemetry/core": "2.6.1", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.6.1.tgz", + "integrity": "sha512-9t9hJHX15meBy2NmTJxL+NJfXmnausR2xUDvE19XQce0Qi/GBtDGamU8nS1RMbdgDmhgpm3VaOu2+fiS/SfTpQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/resources": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-node/-/sdk-node-0.214.0.tgz", + "integrity": "sha512-gl2XvQBJuPjhGcw9SsnQO5qxChAPMuGRPFaD8lqtF+Cey91NgGUQ0sio2vlDFOSm3JOLzc44vL+OAfx1dXuZjg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.214.0", + "@opentelemetry/configuration": "0.214.0", + "@opentelemetry/context-async-hooks": "2.6.1", + "@opentelemetry/core": "2.6.1", + "@opentelemetry/exporter-logs-otlp-grpc": "0.214.0", + "@opentelemetry/exporter-logs-otlp-http": "0.214.0", + "@opentelemetry/exporter-logs-otlp-proto": "0.214.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "0.214.0", + "@opentelemetry/exporter-metrics-otlp-http": "0.214.0", + "@opentelemetry/exporter-metrics-otlp-proto": "0.214.0", + "@opentelemetry/exporter-prometheus": "0.214.0", + "@opentelemetry/exporter-trace-otlp-grpc": "0.214.0", + "@opentelemetry/exporter-trace-otlp-http": "0.214.0", + "@opentelemetry/exporter-trace-otlp-proto": "0.214.0", + "@opentelemetry/exporter-zipkin": "2.6.1", + "@opentelemetry/instrumentation": "0.214.0", + "@opentelemetry/otlp-exporter-base": "0.214.0", + "@opentelemetry/propagator-b3": "2.6.1", + "@opentelemetry/propagator-jaeger": "2.6.1", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/sdk-logs": "0.214.0", + "@opentelemetry/sdk-metrics": "2.6.1", + "@opentelemetry/sdk-trace-base": "2.6.1", + "@opentelemetry/sdk-trace-node": "2.6.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.6.1.tgz", + "integrity": "sha512-r86ut4T1e8vNwB35CqCcKd45yzqH6/6Wzvpk2/cZB8PsPLlZFTvrh8yfOS3CYZYcUmAx4hHTZJ8AO8Dj8nrdhw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/resources": "2.6.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.6.1.tgz", + "integrity": "sha512-Hh2i4FwHWRFhnO2Q/p6svMxy8MPsNCG0uuzUY3glqm0rwM0nQvbTO1dXSp9OqQoTKXcQzaz9q1f65fsurmOhNw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "2.6.1", + "@opentelemetry/core": "2.6.1", + "@opentelemetry/sdk-trace-base": "2.6.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.40.0.tgz", + "integrity": "sha512-cifvXDhcqMwwTlTK04GBNeIe7yyo28Mfby85QXFe1Yk8nmi36Ab/5UQwptOx84SsoGNRg+EVSjwzfSZMy6pmlw==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -1386,6 +1963,70 @@ "node": ">=14" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.52.0", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.52.0.tgz", @@ -1831,7 +2472,6 @@ "version": "20.19.17", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.17.tgz", "integrity": "sha512-gfehUI8N1z92kygssiuWvLiwcbOB3IRktR6hTDgJlXMYh5OvkPSRmgfoBUmfZt+vhwJtX7v1Yw4KvvAf7c5QKQ==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -2529,7 +3169,6 @@ "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "dev": true, "license": "MIT", "bin": { "acorn": "bin/acorn" @@ -2538,6 +3177,15 @@ "node": ">=0.4.0" } }, + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^8" + } + }, "node_modules/acorn-jsx": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", @@ -2595,6 +3243,15 @@ "zod": "^3.25.76 || ^4.1.8" } }, + "node_modules/ai/node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -2628,7 +3285,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -2898,6 +3554,12 @@ "node": ">=8" } }, + "node_modules/cjs-module-lexer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", + "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", + "license": "MIT" + }, "node_modules/clean-regexp": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/clean-regexp/-/clean-regexp-1.0.0.tgz", @@ -2948,6 +3610,78 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/color": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/color/-/color-5.0.3.tgz", @@ -2965,7 +3699,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -2978,7 +3711,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, "license": "MIT" }, "node_modules/color-string": { @@ -3099,7 +3831,6 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -3224,7 +3955,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -3916,6 +4646,15 @@ "setimmediate": "^1.0.5" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-east-asian-width": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", @@ -4084,6 +4823,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/import-in-the-middle": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-3.0.1.tgz", + "integrity": "sha512-pYkiyXVL2Mf3pozdlDGV6NAObxQx13Ae8knZk1UJRJ6uRW/ZRmTGHlQYtrsSl7ubuE5F8CD1z+s1n4RHNuTtuA==", + "license": "Apache-2.0", + "dependencies": { + "acorn": "^8.15.0", + "acorn-import-attributes": "^1.9.5", + "cjs-module-lexer": "^2.2.0", + "module-details-from-path": "^1.0.4" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -4152,7 +4906,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4371,6 +5124,30 @@ "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", "license": "MIT" }, + "node_modules/langfuse": { + "version": "3.38.20", + "resolved": "https://registry.npmjs.org/langfuse/-/langfuse-3.38.20.tgz", + "integrity": "sha512-MAmBAASSzJtmK1O9HQegA1mFsQhT8Yf+OJRGvE7FXkyv3g/eiBE0glLD0Ohg3pkxhoPdggM5SejK7ue9ctlaMA==", + "license": "MIT", + "dependencies": { + "langfuse-core": "^3.38.20" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/langfuse-core": { + "version": "3.38.20", + "resolved": "https://registry.npmjs.org/langfuse-core/-/langfuse-core-3.38.20.tgz", + "integrity": "sha512-zBKVmQN/1oT5VWZUBYlWzvokIlkC/6mnpgr/2atMyTeAm+jR3ia7w2iJMjlrF5/oG8ukO1s8+LDRCzJpF1QeEA==", + "license": "MIT", + "dependencies": { + "mustache": "^4.2.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -4437,6 +5214,12 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "license": "MIT" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -4496,6 +5279,12 @@ "node": ">= 12.0.0" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/loupe": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", @@ -4625,12 +5414,27 @@ "dev": true, "license": "MIT" }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -5063,6 +5867,30 @@ "node": ">= 0.8.0" } }, + "node_modules/protobufjs": { + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -5157,6 +5985,28 @@ "node": ">=6" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-in-the-middle": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", + "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "module-details-from-path": "^1.0.3" + }, + "engines": { + "node": ">=9.3.0 || >=8.10.0 <9.0.0" + } + }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -6072,7 +6922,6 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, "license": "MIT" }, "node_modules/unrs-resolver": { @@ -6910,6 +7759,15 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, "node_modules/yaml": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz", @@ -6922,6 +7780,74 @@ "node": ">= 14.6" } }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/yocto-queue": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", diff --git a/package.json b/package.json index 284ed39..de38d92 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "url": "https://github.com/TRocket-Labs/vectorlint/issues" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.6.0" }, "dependencies": { "@ai-sdk/amazon-bedrock": "^4.0.64", @@ -60,12 +60,16 @@ "@ai-sdk/google": "^3.0.31", "@ai-sdk/openai": "^3.0.33", "@ai-sdk/perplexity": "^1.0.0", + "@langfuse/otel": "^5.1.0", + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/sdk-node": "^0.214.0", "@types/micromatch": "^4.0.9", "ai": "^6.0.99", "chalk": "^5.3.0", "commander": "^12.0.0", "fast-glob": "^3.3.2", "fuzzball": "^2.2.3", + "langfuse": "^3.38.20", "micromatch": "^4.0.5", "ora": "^8.2.0", "p-limit": "^7.3.0", diff --git a/src/boundaries/env-parser.ts b/src/boundaries/env-parser.ts index a31340e..02ecd0b 100644 --- a/src/boundaries/env-parser.ts +++ b/src/boundaries/env-parser.ts @@ -66,7 +66,7 @@ function formatProviderValidationError(zodError: z.ZodError, env: unknown): stri const missingLangfuseFields = [...new Set(langfuseFields)]; if (missingLangfuseFields.length > 0) { - return `Missing required Langfuse observability environment variables: ${missingLangfuseFields.join(', ')}. When using OBSERVABILITY_BACKEND=langfuse, ensure LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and LANGFUSE_BASE_URL are set.`; + return `Missing required Langfuse observability environment variables: ${missingLangfuseFields.join(', ')}. When using OBSERVABILITY_BACKEND=langfuse, ensure LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY are set.`; } } diff --git a/src/cli/commands.ts b/src/cli/commands.ts index ecc8b22..5e30748 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -21,6 +21,10 @@ import { evaluateFiles } from './orchestrator'; import { DEFAULT_REVIEW_MODE, OUTPUT_FORMATS, OutputFormat } from './types'; import { DEFAULT_CONFIG_FILENAME, USER_INSTRUCTION_FILENAME } from '../config/constants'; import { createWinstonLogger } from '../logging/winston-logger'; +import type { AIObservability } from '../observability/ai-observability'; +import { createObservability } from '../observability/factory'; +import { NoopObservability } from '../observability/noop-observability'; +import type { Logger } from '../logging/logger'; // eslint-disable-next-line @typescript-eslint/naming-convention const __filename = fileURLToPath(import.meta.url); @@ -97,7 +101,7 @@ export function registerMainCommand(program: Command): void { extraLines: ['Please set these in your .env file or environment.'], }); - // Load directive and create provider + // Load directive and prompt inputs before provider setup const directive = await runOrExit('Loading directive', () => loadDirective()); // Load user instructions (VECTORLINT.md) @@ -110,22 +114,6 @@ export function registerMainCommand(program: Command): void { level: cliOptions.verbose ? 'debug' : 'info', }); - const provider = createProvider( - env, - { - debug: cliOptions.verbose, - showPrompt: cliOptions.showPrompt, - showPromptTrunc: cliOptions.showPromptTrunc, - logger: runtimeLogger, - }, - new DefaultRequestBuilder(directive, userInstructions.content || undefined) - ); - - if (cliOptions.verbose) { - const directiveLen = directive ? directive.length : 0; - console.log(`[vectorlint] Directive active: ${directiveLen} char(s)`); - } - // Load config and prompts const config = await runOrExit('Loading configuration', () => loadConfig(process.cwd(), cliOptions.config)); @@ -190,47 +178,93 @@ export function registerMainCommand(program: Command): void { process.exit(1); } + if (cliOptions.verbose) { + const directiveLen = directive ? directive.length : 0; + console.log(`[vectorlint] Directive active: ${directiveLen} char(s)`); + } + let observability: AIObservability = new NoopObservability(); + let exitCode = 1; - // Create search provider if API key is available - const searchProvider: SearchProvider | undefined = process.env.PERPLEXITY_API_KEY - ? new PerplexitySearchProvider({ logger: runtimeLogger }) - : undefined; - - // Run evaluations via orchestrator - const result = await evaluateFiles(targets, { - prompts, - rulesPath, - provider, - ...(searchProvider ? { searchProvider } : {}), - concurrency: config.concurrency, - verbose: cliOptions.verbose, - debugJson: cliOptions.debugJson, - outputFormat: cliOptions.output, - mode: cliOptions.mode, - printMode: cliOptions.print, - scanPaths: config.scanPaths, - pricing: { - inputPricePerMillion: env.INPUT_PRICE_PER_MILLION, - outputPricePerMillion: env.OUTPUT_PRICE_PER_MILLION, - }, - ...(userInstructions.content ? { userInstructionContent: userInstructions.content } : {}), - }); + try { + observability = await initializeObservability(env, runtimeLogger); - // Print global summary (only for line format) - if (cliOptions.output === OutputFormat.Line) { - if (result.tokenUsage) { - printTokenUsage(result.tokenUsage); - } - printGlobalSummary( - result.totalFiles, - result.totalErrors, - result.totalWarnings, - result.requestFailures + const provider = createProvider( + env, + { + debug: cliOptions.verbose, + showPrompt: cliOptions.showPrompt, + showPromptTrunc: cliOptions.showPromptTrunc, + logger: runtimeLogger, + observability, + }, + new DefaultRequestBuilder(directive, userInstructions.content || undefined) ); + + // Create search provider if API key is available + const searchProvider: SearchProvider | undefined = process.env.PERPLEXITY_API_KEY + ? new PerplexitySearchProvider({ logger: runtimeLogger }) + : undefined; + + // Run evaluations via orchestrator + const result = await evaluateFiles(targets, { + prompts, + rulesPath, + provider, + ...(searchProvider ? { searchProvider } : {}), + concurrency: config.concurrency, + verbose: cliOptions.verbose, + debugJson: cliOptions.debugJson, + outputFormat: cliOptions.output, + mode: cliOptions.mode, + printMode: cliOptions.print, + scanPaths: config.scanPaths, + pricing: { + inputPricePerMillion: env.INPUT_PRICE_PER_MILLION, + outputPricePerMillion: env.OUTPUT_PRICE_PER_MILLION, + }, + ...(userInstructions.content ? { userInstructionContent: userInstructions.content } : {}), + }); + + // Print global summary (only for line format) + if (cliOptions.output === OutputFormat.Line) { + if (result.tokenUsage) { + printTokenUsage(result.tokenUsage); + } + printGlobalSummary( + result.totalFiles, + result.totalErrors, + result.totalWarnings, + result.requestFailures + ); + } + + exitCode = result.hadOperationalErrors || result.hadSeverityErrors ? 1 : 0; + } finally { + try { + await observability.shutdown?.(); + } catch (error) { + const err = handleUnknownError(error, 'Shutting down observability'); + runtimeLogger.warn('[vectorlint] Observability shutdown failed', { + error: err.message, + }); + } } - // Exit with appropriate code - process.exit(result.hadOperationalErrors || result.hadSeverityErrors ? 1 : 0); + process.exit(exitCode); }); } + +async function initializeObservability(env: ReturnType, logger: Logger): Promise { + try { + const observability = createObservability(env, logger); + await observability.init(); + return observability; + } catch (error) { + const err = handleUnknownError(error, 'Initializing observability'); + logger.warn('[vectorlint] Observability initialization failed; continuing without telemetry', { + error: err.message, + }); + return new NoopObservability(); + } +} diff --git a/src/observability/factory.ts b/src/observability/factory.ts index f066994..c70a54d 100644 --- a/src/observability/factory.ts +++ b/src/observability/factory.ts @@ -9,14 +9,14 @@ export function createObservability(env: EnvConfig, logger?: Logger): AIObservab return new NoopObservability(); } - if (!env.LANGFUSE_PUBLIC_KEY || !env.LANGFUSE_SECRET_KEY || !env.LANGFUSE_BASE_URL) { - throw new Error('Langfuse observability requires LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and LANGFUSE_BASE_URL'); + if (!env.LANGFUSE_PUBLIC_KEY || !env.LANGFUSE_SECRET_KEY) { + throw new Error('Langfuse observability requires LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY'); } return new LangfuseObservability({ publicKey: env.LANGFUSE_PUBLIC_KEY, secretKey: env.LANGFUSE_SECRET_KEY, - baseUrl: env.LANGFUSE_BASE_URL, + ...(env.LANGFUSE_BASE_URL ? { baseUrl: env.LANGFUSE_BASE_URL } : {}), logger, }); } diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index 5035b14..bba39c4 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -1,21 +1,89 @@ -import type { Logger } from '../logging/logger'; +import { LangfuseSpanProcessor } from '@langfuse/otel'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { createNoopLogger, type Logger } from '../logging/logger'; import type { AIExecutionContext, AIObservability } from './ai-observability'; export interface LangfuseObservabilityConfig { publicKey: string; secretKey: string; - baseUrl: string; + baseUrl?: string; logger?: Logger; } export class LangfuseObservability implements AIObservability { - constructor(private readonly _config: LangfuseObservabilityConfig) {} + private sdk?: NodeSDK; + private initPromise?: Promise; + private readonly logger: Logger; - init(): void {} + constructor(private readonly config: LangfuseObservabilityConfig) { + this.logger = config.logger ?? createNoopLogger(); + } + + async init(): Promise { + if (this.initPromise) { + return this.initPromise; + } + + if (this.sdk) { + return; + } + + this.initPromise = Promise.resolve().then(() => { + const spanProcessor = new LangfuseSpanProcessor({ + publicKey: this.config.publicKey, + secretKey: this.config.secretKey, + ...(this.config.baseUrl ? { baseUrl: this.config.baseUrl } : {}), + }); + + const sdk = new NodeSDK({ + spanProcessors: [spanProcessor], + }); + + sdk.start(); + this.sdk = sdk; + }); - decorateCall(_context: AIExecutionContext): Record { - return {}; + try { + await this.initPromise; + } catch (error) { + this.initPromise = undefined; + throw error; + } } - shutdown(): void {} + decorateCall(context: AIExecutionContext): Record { + return { + experimental_telemetry: { + isEnabled: true, + functionId: `vectorlint.${context.operation}`, + metadata: { + provider: context.provider, + model: context.model, + ...(context.evaluator ? { evaluator: context.evaluator } : {}), + ...(context.rule ? { rule: context.rule } : {}), + }, + recordInputs: true, + recordOutputs: true, + }, + }; + } + + async shutdown(): Promise { + const sdk = this.sdk; + this.sdk = undefined; + this.initPromise = undefined; + + if (!sdk) { + return; + } + + try { + await sdk.shutdown(); + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + this.logger.warn('[vectorlint] Failed to shutdown Langfuse observability SDK', { + error: err.message, + }); + } + } } diff --git a/src/observability/noop-observability.ts b/src/observability/noop-observability.ts index 83f0e09..e7cc8f7 100644 --- a/src/observability/noop-observability.ts +++ b/src/observability/noop-observability.ts @@ -3,7 +3,8 @@ import type { AIExecutionContext, AIObservability } from './ai-observability'; export class NoopObservability implements AIObservability { init(): void {} - decorateCall(_context: AIExecutionContext): Record { + decorateCall(context: AIExecutionContext): Record { + void context; return {}; } diff --git a/src/schemas/env-schemas.ts b/src/schemas/env-schemas.ts index fa9cea7..76cd78f 100644 --- a/src/schemas/env-schemas.ts +++ b/src/schemas/env-schemas.ts @@ -111,14 +111,6 @@ export const ENV_SCHEMA = z.discriminatedUnion('LLM_PROVIDER', [ message: 'LANGFUSE_SECRET_KEY is required when OBSERVABILITY_BACKEND=langfuse', }); } - - if (!data.LANGFUSE_BASE_URL) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['LANGFUSE_BASE_URL'], - message: 'LANGFUSE_BASE_URL is required when OBSERVABILITY_BACKEND=langfuse', - }); - } } }); diff --git a/tests/env-parser.test.ts b/tests/env-parser.test.ts index a4faf69..65f456b 100644 --- a/tests/env-parser.test.ts +++ b/tests/env-parser.test.ts @@ -331,7 +331,7 @@ describe('Environment Parser', () => { }; expect(() => parseEnvironment(env)).toThrow(ValidationError); - expect(() => parseEnvironment(env)).toThrow(/Missing required Langfuse observability environment variables.*LANGFUSE_PUBLIC_KEY.*LANGFUSE_SECRET_KEY.*LANGFUSE_BASE_URL/); + expect(() => parseEnvironment(env)).toThrow(/Missing required Langfuse observability environment variables.*LANGFUSE_PUBLIC_KEY.*LANGFUSE_SECRET_KEY/); }); }); }); diff --git a/tests/main-command-observability.test.ts b/tests/main-command-observability.test.ts new file mode 100644 index 0000000..178a032 --- /dev/null +++ b/tests/main-command-observability.test.ts @@ -0,0 +1,191 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { Command } from 'commander'; +import { createMockLogger } from './utils'; +import { NoopObservability } from '../src/observability/noop-observability'; + +const MOCK_PARSE_CLI_OPTIONS = vi.hoisted(() => vi.fn()); +const MOCK_PARSE_ENVIRONMENT = vi.hoisted(() => vi.fn()); +const MOCK_LOAD_DIRECTIVE = vi.hoisted(() => vi.fn()); +const MOCK_LOAD_USER_INSTRUCTIONS = vi.hoisted(() => vi.fn()); +const MOCK_CREATE_WINSTON_LOGGER = vi.hoisted(() => vi.fn()); +const MOCK_LOAD_CONFIG = vi.hoisted(() => vi.fn()); +const MOCK_RESOLVE_TARGETS = vi.hoisted(() => vi.fn()); +const MOCK_CREATE_OBSERVABILITY = vi.hoisted(() => vi.fn()); +const MOCK_CREATE_PROVIDER = vi.hoisted(() => vi.fn()); +const MOCK_EVALUATE_FILES = vi.hoisted(() => vi.fn()); +const MOCK_LOAD_RULE_FILE = vi.hoisted(() => vi.fn()); +const MOCK_LIST_ALL_PACKS = vi.hoisted(() => vi.fn()); +const MOCK_FIND_RULE_FILES = vi.hoisted(() => vi.fn()); + +vi.mock('../src/boundaries/index', () => ({ + parseCliOptions: MOCK_PARSE_CLI_OPTIONS, + parseEnvironment: MOCK_PARSE_ENVIRONMENT, +})); + +vi.mock('../src/prompts/directive-loader', () => ({ + loadDirective: MOCK_LOAD_DIRECTIVE, +})); + +vi.mock('../src/boundaries/user-instruction-loader', () => ({ + loadUserInstructions: MOCK_LOAD_USER_INSTRUCTIONS, +})); + +vi.mock('../src/logging/winston-logger', () => ({ + createWinstonLogger: MOCK_CREATE_WINSTON_LOGGER, +})); + +vi.mock('../src/boundaries/config-loader', () => ({ + loadConfig: MOCK_LOAD_CONFIG, +})); + +vi.mock('../src/scan/file-resolver', () => ({ + resolveTargets: MOCK_RESOLVE_TARGETS, +})); + +vi.mock('../src/observability/factory', () => ({ + createObservability: MOCK_CREATE_OBSERVABILITY, +})); + +vi.mock('../src/providers/provider-factory', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createProvider: MOCK_CREATE_PROVIDER, + }; +}); + +vi.mock('../src/cli/orchestrator', () => ({ + evaluateFiles: MOCK_EVALUATE_FILES, +})); + +vi.mock('../src/prompts/prompt-loader', () => ({ + loadRuleFile: MOCK_LOAD_RULE_FILE, +})); + +vi.mock('../src/config/preset-loader', () => ({ + PresetLoader: class PresetLoader {}, +})); + +vi.mock('../src/boundaries/rule-pack-loader', () => ({ + RulePackLoader: class RulePackLoader { + listAllPacks() { + return Promise.resolve(MOCK_LIST_ALL_PACKS()); + } + + findRuleFiles() { + return Promise.resolve(MOCK_FIND_RULE_FILES()); + } + }, +})); + +describe('Main command observability lifecycle', () => { + const env = { + LLM_PROVIDER: 'openai', + OPENAI_API_KEY: 'sk-test', + OPENAI_MODEL: 'gpt-4o', + }; + + let exitSpy: ReturnType; + const runtimeLogger = createMockLogger(); + + beforeEach(() => { + vi.clearAllMocks(); + + MOCK_PARSE_CLI_OPTIONS.mockResolvedValue({ + verbose: false, + showPrompt: false, + showPromptTrunc: false, + debugJson: false, + output: 'line', + mode: 'standard', + print: false, + config: undefined, + }); + MOCK_PARSE_ENVIRONMENT.mockReturnValue(env); + MOCK_LOAD_DIRECTIVE.mockResolvedValue(''); + MOCK_LOAD_USER_INSTRUCTIONS.mockReturnValue({ content: '', tokenEstimate: 0 }); + MOCK_CREATE_WINSTON_LOGGER.mockReturnValue(runtimeLogger); + MOCK_LOAD_CONFIG.mockResolvedValue({ + rulesPath: undefined, + concurrency: 1, + scanPaths: [], + configDir: process.cwd(), + }); + MOCK_LIST_ALL_PACKS.mockResolvedValue([]); + MOCK_FIND_RULE_FILES.mockResolvedValue([]); + MOCK_LOAD_RULE_FILE.mockReturnValue({ prompt: undefined, warning: undefined }); + MOCK_RESOLVE_TARGETS.mockReturnValue(['README.md']); + MOCK_CREATE_PROVIDER.mockReturnValue({ mocked: true }); + MOCK_EVALUATE_FILES.mockResolvedValue({ + totalFiles: 1, + totalErrors: 0, + totalWarnings: 0, + requestFailures: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + tokenUsage: undefined, + }); + + exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code?: string | number | null) => { + throw new Error(`process.exit:${code ?? ''}`); + }) as never); + }); + + afterEach(() => { + exitSpy.mockRestore(); + }); + + it('initializes observability before creating the provider and shuts it down before exit', async () => { + const observability = { + init: vi.fn().mockResolvedValue(undefined), + decorateCall: vi.fn(() => ({})), + shutdown: vi.fn().mockResolvedValue(undefined), + }; + MOCK_CREATE_OBSERVABILITY.mockReturnValue(observability); + + const { registerMainCommand } = await import('../src/cli/commands'); + const program = new Command(); + registerMainCommand(program); + + await expect(program.parseAsync(['node', 'test', 'README.md'])).rejects.toThrow('process.exit:0'); + + expect(MOCK_CREATE_OBSERVABILITY).toHaveBeenCalledWith(env, runtimeLogger); + expect(observability.init).toHaveBeenCalledTimes(1); + expect(MOCK_CREATE_PROVIDER).toHaveBeenCalledWith( + env, + expect.objectContaining({ + logger: runtimeLogger, + observability, + }), + expect.anything() + ); + expect(observability.init.mock.invocationCallOrder[0]).toBeLessThan(MOCK_CREATE_PROVIDER.mock.invocationCallOrder[0]); + expect(observability.shutdown).toHaveBeenCalledTimes(1); + expect(observability.shutdown.mock.invocationCallOrder[0]).toBeLessThan(exitSpy.mock.invocationCallOrder[0]); + }); + + it('falls back to noop observability when initialization fails', async () => { + const failingObservability = { + init: vi.fn().mockRejectedValue(new Error('boom')), + decorateCall: vi.fn(() => ({})), + shutdown: vi.fn().mockResolvedValue(undefined), + }; + MOCK_CREATE_OBSERVABILITY.mockReturnValue(failingObservability); + + const { registerMainCommand } = await import('../src/cli/commands'); + const program = new Command(); + registerMainCommand(program); + + await expect(program.parseAsync(['node', 'test', 'README.md'])).rejects.toThrow('process.exit:0'); + + expect(runtimeLogger.warn).toHaveBeenCalledWith( + '[vectorlint] Observability initialization failed; continuing without telemetry', + expect.objectContaining({ + error: 'boom', + }) + ); + + const providerOptions = MOCK_CREATE_PROVIDER.mock.calls.at(-1)?.[1] as { observability?: unknown }; + expect(providerOptions.observability).toBeInstanceOf(NoopObservability); + }); +}); diff --git a/tests/observability/factory.test.ts b/tests/observability/factory.test.ts index 5db00ee..759a4f6 100644 --- a/tests/observability/factory.test.ts +++ b/tests/observability/factory.test.ts @@ -46,4 +46,15 @@ describe('createObservability', () => { expect(() => createObservability(env)).toThrow(/Langfuse observability requires/); }); + + it('allows langfuse backend without explicit base URL', () => { + const env: EnvConfig = { + ...baseEnv, + OBSERVABILITY_BACKEND: 'langfuse', + LANGFUSE_PUBLIC_KEY: 'pk-lf-test', + LANGFUSE_SECRET_KEY: 'sk-lf-test', + }; + + expect(createObservability(env)).toBeInstanceOf(LangfuseObservability); + }); }); diff --git a/tests/observability/langfuse-observability.test.ts b/tests/observability/langfuse-observability.test.ts new file mode 100644 index 0000000..5644848 --- /dev/null +++ b/tests/observability/langfuse-observability.test.ts @@ -0,0 +1,127 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createMockLogger } from '../utils'; + +const START_MOCK = vi.hoisted(() => vi.fn()); +const SHUTDOWN_MOCK = vi.hoisted(() => vi.fn()); +const NODE_SDK_CTOR_MOCK = vi.hoisted(() => vi.fn(() => ({ + start: START_MOCK, + shutdown: SHUTDOWN_MOCK, +}))); +const LANGFUSE_SPAN_PROCESSOR_CTOR_MOCK = vi.hoisted(() => vi.fn(() => ({ mocked: true }))); + +vi.mock('@opentelemetry/sdk-node', () => ({ + NodeSDK: NODE_SDK_CTOR_MOCK, +})); + +vi.mock('@langfuse/otel', () => ({ + LangfuseSpanProcessor: LANGFUSE_SPAN_PROCESSOR_CTOR_MOCK, +})); + +import { LangfuseObservability } from '../../src/observability/langfuse-observability'; + +describe('LangfuseObservability', () => { + beforeEach(() => { + vi.clearAllMocks(); + START_MOCK.mockImplementation(() => undefined); + SHUTDOWN_MOCK.mockResolvedValue(undefined); + }); + + it('returns AI SDK telemetry options with full payload recording enabled', () => { + const subject = new LangfuseObservability({ + publicKey: 'pk-lf-test', + secretKey: 'sk-lf-test', + baseUrl: 'https://cloud.langfuse.com', + }); + + expect(subject.decorateCall({ + operation: 'structured-eval', + provider: 'openai', + model: 'gpt-4o', + evaluator: 'clarity', + rule: 'no-fluff', + })).toEqual({ + experimental_telemetry: { + isEnabled: true, + functionId: 'vectorlint.structured-eval', + metadata: { + provider: 'openai', + model: 'gpt-4o', + evaluator: 'clarity', + rule: 'no-fluff', + }, + recordInputs: true, + recordOutputs: true, + }, + }); + }); + + it('starts OTEL only once when init is called multiple times', async () => { + const subject = new LangfuseObservability({ + publicKey: 'pk-lf-test', + secretKey: 'sk-lf-test', + baseUrl: 'https://cloud.langfuse.com', + }); + + await subject.init(); + await subject.init(); + + expect(LANGFUSE_SPAN_PROCESSOR_CTOR_MOCK).toHaveBeenCalledTimes(1); + expect(LANGFUSE_SPAN_PROCESSOR_CTOR_MOCK).toHaveBeenCalledWith({ + publicKey: 'pk-lf-test', + secretKey: 'sk-lf-test', + baseUrl: 'https://cloud.langfuse.com', + }); + + expect(NODE_SDK_CTOR_MOCK).toHaveBeenCalledTimes(1); + expect(NODE_SDK_CTOR_MOCK).toHaveBeenCalledWith( + expect.objectContaining({ + spanProcessors: [expect.any(Object)], + }) + ); + expect(START_MOCK).toHaveBeenCalledTimes(1); + }); + + it('shuts down the SDK when initialized', async () => { + const subject = new LangfuseObservability({ + publicKey: 'pk-lf-test', + secretKey: 'sk-lf-test', + baseUrl: 'https://cloud.langfuse.com', + }); + + await subject.init(); + await subject.shutdown(); + + expect(SHUTDOWN_MOCK).toHaveBeenCalledTimes(1); + }); + + it('does nothing when shutdown is called before init', async () => { + const subject = new LangfuseObservability({ + publicKey: 'pk-lf-test', + secretKey: 'sk-lf-test', + baseUrl: 'https://cloud.langfuse.com', + }); + + await expect(subject.shutdown()).resolves.toBeUndefined(); + expect(SHUTDOWN_MOCK).not.toHaveBeenCalled(); + }); + + it('logs and continues when SDK shutdown fails', async () => { + SHUTDOWN_MOCK.mockRejectedValueOnce(new Error('shutdown failed')); + const logger = createMockLogger(); + const subject = new LangfuseObservability({ + publicKey: 'pk-lf-test', + secretKey: 'sk-lf-test', + baseUrl: 'https://cloud.langfuse.com', + logger, + }); + + await subject.init(); + await expect(subject.shutdown()).resolves.toBeUndefined(); + expect(logger.warn).toHaveBeenCalledWith( + '[vectorlint] Failed to shutdown Langfuse observability SDK', + expect.objectContaining({ + error: 'shutdown failed', + }) + ); + }); +}); From 7f5924abd51f1855ff443ea5667757ddeb0335c4 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 15 Apr 2026 10:54:56 +0100 Subject: [PATCH 4/6] docs(observability): document Langfuse configuration --- .env.example | 8 +++++++- CONFIGURATION.md | 18 ++++++++++++++++++ README.md | 21 +++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 542281a..21c90d1 100644 --- a/.env.example +++ b/.env.example @@ -29,4 +29,10 @@ # Search Provider Configuration # SEARCH_PROVIDER=perplexity -# PERPLEXITY_API_KEY=pplx-0000000000000000 \ No newline at end of file +# PERPLEXITY_API_KEY=pplx-0000000000000000 + +# Observability Configuration (optional) +# OBSERVABILITY_BACKEND=langfuse +# LANGFUSE_PUBLIC_KEY=pk-lf-0000000000000000 +# LANGFUSE_SECRET_KEY=sk-lf-0000000000000000 +# LANGFUSE_BASE_URL=https://cloud.langfuse.com diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 27cbb59..34dfcf8 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -99,6 +99,24 @@ SEARCH_PROVIDER=perplexity PERPLEXITY_API_KEY=pplx-... ``` +### Observability + +VectorLint can optionally emit AI execution telemetry to Langfuse. The first implementation is scoped to the Vercel AI SDK calls made through `VercelAIProvider`. + +**Example configuration for Langfuse:** + +```bash +OBSERVABILITY_BACKEND=langfuse +LANGFUSE_PUBLIC_KEY=pk-lf-... +LANGFUSE_SECRET_KEY=sk-lf-... +# Optional for self-hosted Langfuse. Defaults to cloud.langfuse.com. +LANGFUSE_BASE_URL=https://cloud.langfuse.com +``` + +- Observability is best-effort and non-blocking. +- If initialization or shutdown fails, VectorLint logs a warning and continues. +- Prompts and outputs are recorded when Langfuse observability is enabled. + ### False-Positive Filtering (PAT) VectorLint uses PAT (Pay A Tax) style gate checks to reduce false positives. The model may return many raw candidates, but only candidates that pass deterministic gate checks are surfaced in CLI output. diff --git a/README.md b/README.md index 0c2fe3f..08a5140 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ Install globally from npm: npm install -g vectorlint ``` +VectorLint currently requires Node.js `20.6+`. + Verify installation: ```bash @@ -123,6 +125,25 @@ VectorLint is bundled with a `VectorLint` preset containing rules for AI pattern 👉 **[Learn how to create custom rules →](./CREATING_RULES.md)** +### 4. Optional: Enable Langfuse observability + +VectorLint can emit AI execution telemetry through Langfuse without hardcoding Langfuse into the provider layer. This is best-effort instrumentation for the Vercel AI SDK calls used by `VercelAIProvider`. + +Add these environment variables to your global config or local `.env` file: + +```toml +[env] +OBSERVABILITY_BACKEND = "langfuse" +LANGFUSE_PUBLIC_KEY = "pk-lf-..." +LANGFUSE_SECRET_KEY = "sk-lf-..." +# Optional for self-hosted Langfuse. Defaults to cloud.langfuse.com. +LANGFUSE_BASE_URL = "https://cloud.langfuse.com" +``` + +Notes: +- Observability is non-blocking. If Langfuse setup fails, VectorLint continues without telemetry. +- Prompts and outputs are recorded when Langfuse observability is enabled. + ## Agent Mode Agent mode is for reviews that need context from several files such as From 2ab0f5a75ec8c6ec63b38b4f2cf4a9b5b9a830dd Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 15 Apr 2026 11:10:46 +0100 Subject: [PATCH 5/6] refactor(observability): simplify error handling and deps --- package-lock.json | 36 +-------------------- package.json | 1 - src/boundaries/env-parser.ts | 4 ++- src/observability/langfuse-observability.ts | 3 +- src/providers/vercel-ai-provider.ts | 3 +- tests/env-parser.test.ts | 2 +- 6 files changed, 9 insertions(+), 40 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3fc9064..49e625d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,6 @@ "commander": "^12.0.0", "fast-glob": "^3.3.2", "fuzzball": "^2.2.3", - "langfuse": "^3.38.20", "micromatch": "^4.0.5", "ora": "^8.2.0", "p-limit": "^7.3.0", @@ -56,7 +55,7 @@ "vitest": "^2.0.0" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.6.0" } }, "node_modules/@ai-sdk/amazon-bedrock": { @@ -5124,30 +5123,6 @@ "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", "license": "MIT" }, - "node_modules/langfuse": { - "version": "3.38.20", - "resolved": "https://registry.npmjs.org/langfuse/-/langfuse-3.38.20.tgz", - "integrity": "sha512-MAmBAASSzJtmK1O9HQegA1mFsQhT8Yf+OJRGvE7FXkyv3g/eiBE0glLD0Ohg3pkxhoPdggM5SejK7ue9ctlaMA==", - "license": "MIT", - "dependencies": { - "langfuse-core": "^3.38.20" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/langfuse-core": { - "version": "3.38.20", - "resolved": "https://registry.npmjs.org/langfuse-core/-/langfuse-core-3.38.20.tgz", - "integrity": "sha512-zBKVmQN/1oT5VWZUBYlWzvokIlkC/6mnpgr/2atMyTeAm+jR3ia7w2iJMjlrF5/oG8ukO1s8+LDRCzJpF1QeEA==", - "license": "MIT", - "dependencies": { - "mustache": "^4.2.0" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -5426,15 +5401,6 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, - "node_modules/mustache": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", - "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", - "license": "MIT", - "bin": { - "mustache": "bin/mustache" - } - }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", diff --git a/package.json b/package.json index de38d92..74f474d 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,6 @@ "commander": "^12.0.0", "fast-glob": "^3.3.2", "fuzzball": "^2.2.3", - "langfuse": "^3.38.20", "micromatch": "^4.0.5", "ora": "^8.2.0", "p-limit": "^7.3.0", diff --git a/src/boundaries/env-parser.ts b/src/boundaries/env-parser.ts index 02ecd0b..01c39a6 100644 --- a/src/boundaries/env-parser.ts +++ b/src/boundaries/env-parser.ts @@ -1,6 +1,7 @@ import { z } from 'zod'; import { ENV_SCHEMA, type EnvConfig } from '../schemas/env-schemas'; import { ValidationError, handleUnknownError } from '../errors/index'; +import { ProviderType } from '../providers/provider-factory'; export function parseEnvironment(env: unknown = process.env): EnvConfig { try { @@ -28,7 +29,8 @@ function formatProviderValidationError(zodError: z.ZodError, env: unknown): stri ); if (discriminatorIssue) { - return `LLM_PROVIDER is required and must be one of 'azure-openai', 'anthropic', 'openai', 'gemini', or 'amazon-bedrock'. Received: ${providerType ?? 'undefined'}`; + const allowedProviders = Object.values(ProviderType).map(value => `'${value}'`).join(', '); + return `LLM_PROVIDER is required and must be one of ${allowedProviders}. Received: ${providerType ?? 'undefined'}`; } // Check for missing required fields based on provider type diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index bba39c4..b889812 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -2,6 +2,7 @@ import { LangfuseSpanProcessor } from '@langfuse/otel'; import { NodeSDK } from '@opentelemetry/sdk-node'; import { createNoopLogger, type Logger } from '../logging/logger'; import type { AIExecutionContext, AIObservability } from './ai-observability'; +import { handleUnknownError } from '../errors'; export interface LangfuseObservabilityConfig { publicKey: string; @@ -80,7 +81,7 @@ export class LangfuseObservability implements AIObservability { try { await sdk.shutdown(); } catch (error) { - const err = error instanceof Error ? error : new Error(String(error)); + const err = handleUnknownError(error, 'Shutting down Langfuse observability SDK'); this.logger.warn('[vectorlint] Failed to shutdown Langfuse observability SDK', { error: err.message, }); diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index dad0827..599364d 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -6,6 +6,7 @@ import { AgentToolLoopParams, AgentToolLoopResult, LLMProvider, LLMResult } from import { DefaultRequestBuilder, RequestBuilder } from './request-builder'; import { createNoopLogger, type Logger } from '../logging/logger'; import type { AIExecutionContext, AIObservability } from '../observability/ai-observability'; +import { handleUnknownError } from '../errors'; export interface VercelAIConfig { model: LanguageModel; @@ -202,7 +203,7 @@ export class VercelAIProvider implements LLMProvider { try { return this.observability.decorateCall(context); } catch (error) { - const err = error instanceof Error ? error : new Error(String(error)); + const err = handleUnknownError(error, 'Decorating AI call for observability'); this.logger.warn('[vectorlint] Failed to decorate AI call for observability; continuing without telemetry options', { error: err.message, operation: context.operation, diff --git a/tests/env-parser.test.ts b/tests/env-parser.test.ts index 65f456b..87dc6e1 100644 --- a/tests/env-parser.test.ts +++ b/tests/env-parser.test.ts @@ -208,7 +208,7 @@ describe('Environment Parser', () => { }; expect(() => parseEnvironment(env)).toThrow(ValidationError); - expect(() => parseEnvironment(env)).toThrow(/LLM_PROVIDER is required and must be one of 'azure-openai', 'anthropic', 'openai', 'gemini', or 'amazon-bedrock'/); + expect(() => parseEnvironment(env)).toThrow(/LLM_PROVIDER is required and must be one of 'azure-openai', 'anthropic', 'openai', 'gemini', 'amazon-bedrock'/); }); it('provides specific error message for missing Azure OpenAI variables', () => { From f12b426e39f59b384b86d0c7929ed309e8e72924 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 15 Apr 2026 11:29:53 +0100 Subject: [PATCH 6/6] refactor(observability): tighten telemetry wiring - create the runtime logger only for verbose runs or active telemetry - pass provider and model names explicitly into Vercel AI calls instead of reflecting on SDK model objects - centralize Langfuse backend constants and narrow missing-credential reporting to the required keys - document recorded-data caution and extend tests around logger and telemetry metadata behavior --- README.md | 1 + src/boundaries/env-parser.ts | 10 ++- src/cli/commands.ts | 13 +++- src/observability/factory.ts | 6 +- src/providers/provider-factory.ts | 8 ++ src/providers/vercel-ai-provider.ts | 30 ++------ src/schemas/env-schemas.ts | 7 +- tests/main-command-observability.test.ts | 75 +++++++++++++++++++ tests/provider-factory.test.ts | 15 ++++ .../vercel-ai-provider-agent-loop.test.ts | 4 + tests/vercel-ai-provider.test.ts | 4 + 11 files changed, 139 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 08a5140..859ea51 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ LANGFUSE_BASE_URL = "https://cloud.langfuse.com" Notes: - Observability is non-blocking. If Langfuse setup fails, VectorLint continues without telemetry. - Prompts and outputs are recorded when Langfuse observability is enabled. +- Do not send secrets, credentials, or PII unless your policy explicitly allows observability tooling to access that data. ## Agent Mode diff --git a/src/boundaries/env-parser.ts b/src/boundaries/env-parser.ts index 01c39a6..d6bed0e 100644 --- a/src/boundaries/env-parser.ts +++ b/src/boundaries/env-parser.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { ENV_SCHEMA, type EnvConfig } from '../schemas/env-schemas'; +import { ENV_SCHEMA, OBSERVABILITY_BACKENDS, type EnvConfig } from '../schemas/env-schemas'; import { ValidationError, handleUnknownError } from '../errors/index'; import { ProviderType } from '../providers/provider-factory'; @@ -61,9 +61,13 @@ function formatProviderValidationError(zodError: z.ZodError, env: unknown): stri } } - if (envObj.OBSERVABILITY_BACKEND === 'langfuse') { + if (envObj.OBSERVABILITY_BACKEND === OBSERVABILITY_BACKENDS[0]) { const langfuseFields = issues - .filter((issue) => issue.path.length > 0 && String(issue.path[0]).startsWith('LANGFUSE_')) + .filter((issue) => + issue.code === 'custom' && + issue.path.length > 0 && + ['LANGFUSE_PUBLIC_KEY', 'LANGFUSE_SECRET_KEY'].includes(String(issue.path[0])) + ) .map((issue) => issue.path.join('.')); const missingLangfuseFields = [...new Set(langfuseFields)]; diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 5e30748..369eda1 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -21,6 +21,7 @@ import { evaluateFiles } from './orchestrator'; import { DEFAULT_REVIEW_MODE, OUTPUT_FORMATS, OutputFormat } from './types'; import { DEFAULT_CONFIG_FILENAME, USER_INSTRUCTION_FILENAME } from '../config/constants'; import { createWinstonLogger } from '../logging/winston-logger'; +import { createNoopLogger } from '../logging/logger'; import type { AIObservability } from '../observability/ai-observability'; import { createObservability } from '../observability/factory'; import { NoopObservability } from '../observability/noop-observability'; @@ -110,9 +111,11 @@ export function registerMainCommand(program: Command): void { console.log(`[vectorlint] Loaded user instructions from ${USER_INSTRUCTION_FILENAME} (${userInstructions.tokenEstimate} estimated tokens)`); } - const runtimeLogger = createWinstonLogger({ - level: cliOptions.verbose ? 'debug' : 'info', - }); + const runtimeLogger = shouldUseRuntimeLogger(cliOptions.verbose, env) + ? createWinstonLogger({ + level: cliOptions.verbose ? 'debug' : 'info', + }) + : createNoopLogger(); // Load config and prompts const config = await runOrExit('Loading configuration', () => loadConfig(process.cwd(), cliOptions.config)); @@ -268,3 +271,7 @@ async function initializeObservability(env: ReturnType, return new NoopObservability(); } } + +function shouldUseRuntimeLogger(verbose: boolean, env: ReturnType): boolean { + return verbose || env.OBSERVABILITY_BACKEND === 'langfuse'; +} diff --git a/src/observability/factory.ts b/src/observability/factory.ts index c70a54d..84ec671 100644 --- a/src/observability/factory.ts +++ b/src/observability/factory.ts @@ -1,16 +1,18 @@ import type { Logger } from '../logging/logger'; +import { ValidationError } from '../errors'; import type { EnvConfig } from '../schemas/env-schemas'; +import { OBSERVABILITY_BACKENDS } from '../schemas/env-schemas'; import type { AIObservability } from './ai-observability'; import { LangfuseObservability } from './langfuse-observability'; import { NoopObservability } from './noop-observability'; export function createObservability(env: EnvConfig, logger?: Logger): AIObservability { - if (env.OBSERVABILITY_BACKEND !== 'langfuse') { + if (env.OBSERVABILITY_BACKEND !== OBSERVABILITY_BACKENDS[0]) { return new NoopObservability(); } if (!env.LANGFUSE_PUBLIC_KEY || !env.LANGFUSE_SECRET_KEY) { - throw new Error('Langfuse observability requires LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY'); + throw new ValidationError('Langfuse observability requires LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY'); } return new LangfuseObservability({ diff --git a/src/providers/provider-factory.ts b/src/providers/provider-factory.ts index 83d81b3..3234087 100644 --- a/src/providers/provider-factory.ts +++ b/src/providers/provider-factory.ts @@ -40,6 +40,7 @@ export function createProvider( builder?: RequestBuilder ): LLMProvider { let model: LanguageModel; + let modelName: string; let temperature = 0.2; switch (envConfig.LLM_PROVIDER) { @@ -53,6 +54,7 @@ export function createProvider( // that is not directly assignable to the generic LanguageModel from 'ai'. // Tested with @ai-sdk/azure@1.x — revisit if the SDK adds a typed adapter. model = azure(envConfig.AZURE_OPENAI_DEPLOYMENT_NAME) as unknown as LanguageModel; + modelName = envConfig.AZURE_OPENAI_DEPLOYMENT_NAME; temperature = envConfig.AZURE_OPENAI_TEMPERATURE ?? 0.2; break; } @@ -62,6 +64,7 @@ export function createProvider( apiKey: envConfig.ANTHROPIC_API_KEY, }); model = anthropic(envConfig.ANTHROPIC_MODEL); + modelName = envConfig.ANTHROPIC_MODEL; temperature = envConfig.ANTHROPIC_TEMPERATURE ?? 0.2; break; } @@ -71,6 +74,7 @@ export function createProvider( apiKey: envConfig.OPENAI_API_KEY, }); model = openai(envConfig.OPENAI_MODEL); + modelName = envConfig.OPENAI_MODEL; temperature = envConfig.OPENAI_TEMPERATURE ?? 0.2; break; } @@ -80,6 +84,7 @@ export function createProvider( apiKey: envConfig.GEMINI_API_KEY, }); model = google(envConfig.GEMINI_MODEL); + modelName = envConfig.GEMINI_MODEL; temperature = envConfig.GEMINI_TEMPERATURE ?? 0.2; break; } @@ -91,6 +96,7 @@ export function createProvider( ...(envConfig.AWS_SECRET_ACCESS_KEY && { secretAccessKey: envConfig.AWS_SECRET_ACCESS_KEY }), }); model = bedrock(envConfig.BEDROCK_MODEL) as unknown as LanguageModel; + modelName = envConfig.BEDROCK_MODEL; temperature = envConfig.BEDROCK_TEMPERATURE ?? 0.2; break; } @@ -102,6 +108,8 @@ export function createProvider( const config: VercelAIConfig = { model, + providerName: envConfig.LLM_PROVIDER, + modelName, temperature, ...(envConfig.LLM_PROVIDER === ProviderType.Anthropic && envConfig.ANTHROPIC_MAX_TOKENS !== undefined && { maxTokens: envConfig.ANTHROPIC_MAX_TOKENS }), ...(options.debug !== undefined && { debug: options.debug }), diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 599364d..689a19c 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -10,6 +10,8 @@ import { handleUnknownError } from '../errors'; export interface VercelAIConfig { model: LanguageModel; + providerName?: string; + modelName?: string; temperature?: number; maxTokens?: number; debug?: boolean; @@ -73,8 +75,8 @@ export class VercelAIProvider implements LLMProvider { try { const observabilityOptions = this.getObservabilityOptions({ operation: 'structured-eval', - provider: this.resolveProviderName(), - model: this.resolveModelName(), + provider: this.config.providerName ?? 'unknown', + model: this.config.modelName ?? 'unknown', evaluator: this.extractContextValue(context, 'evaluatorName', 'evaluator'), rule: this.extractContextValue(context, 'ruleName', 'rule'), }); @@ -153,8 +155,8 @@ export class VercelAIProvider implements LLMProvider { ...(params.maxRetries !== undefined ? { maxRetries: params.maxRetries } : {}), ...this.getObservabilityOptions({ operation: 'agent-tool-loop', - provider: this.resolveProviderName(), - model: this.resolveModelName(), + provider: this.config.providerName ?? 'unknown', + model: this.config.modelName ?? 'unknown', }), stopWhen: stepCountIs(params.maxSteps ?? 1000), providerOptions: { @@ -212,26 +214,6 @@ export class VercelAIProvider implements LLMProvider { } } - private resolveProviderName(): string { - const model = this.config.model as unknown as Record; - const provider = model.provider; - if (typeof provider === 'string' && provider.length > 0) { - return provider; - } - return 'unknown'; - } - - private resolveModelName(): string { - const model = this.config.model as unknown as Record; - for (const key of ['modelId', 'model', 'id']) { - const value = model[key]; - if (typeof value === 'string' && value.length > 0) { - return value; - } - } - return 'unknown'; - } - private extractContextValue( context: import('./request-builder').EvalContext | undefined, ...keys: string[] diff --git a/src/schemas/env-schemas.ts b/src/schemas/env-schemas.ts index 76cd78f..777e63f 100644 --- a/src/schemas/env-schemas.ts +++ b/src/schemas/env-schemas.ts @@ -23,6 +23,9 @@ export const BEDROCK_DEFAULT_CONFIG = { model: 'global.anthropic.claude-sonnet-4-5-20250929-v1:0', }; +export const OBSERVABILITY_BACKENDS = ['langfuse'] as const; +export type ObservabilityBackend = (typeof OBSERVABILITY_BACKENDS)[number]; + // Azure OpenAI configuration schema const AZURE_OPENAI_CONFIG_SCHEMA = z.object({ AZURE_OPENAI_API_KEY: z.string().min(1), @@ -64,7 +67,7 @@ const BEDROCK_CONFIG_SCHEMA = z.object({ }); const OBSERVABILITY_ENV_SCHEMA = z.object({ - OBSERVABILITY_BACKEND: z.enum(['langfuse']).optional(), + OBSERVABILITY_BACKEND: z.enum(OBSERVABILITY_BACKENDS).optional(), LANGFUSE_PUBLIC_KEY: z.string().min(1).optional(), LANGFUSE_SECRET_KEY: z.string().min(1).optional(), LANGFUSE_BASE_URL: z.string().url().optional(), @@ -95,7 +98,7 @@ export const ENV_SCHEMA = z.discriminatedUnion('LLM_PROVIDER', [ } } - if (data.OBSERVABILITY_BACKEND === 'langfuse') { + if (data.OBSERVABILITY_BACKEND === OBSERVABILITY_BACKENDS[0]) { if (!data.LANGFUSE_PUBLIC_KEY) { ctx.addIssue({ code: z.ZodIssueCode.custom, diff --git a/tests/main-command-observability.test.ts b/tests/main-command-observability.test.ts index 178a032..359da16 100644 --- a/tests/main-command-observability.test.ts +++ b/tests/main-command-observability.test.ts @@ -136,6 +136,16 @@ describe('Main command observability lifecycle', () => { }); it('initializes observability before creating the provider and shuts it down before exit', async () => { + MOCK_PARSE_CLI_OPTIONS.mockResolvedValue({ + verbose: true, + showPrompt: false, + showPromptTrunc: false, + debugJson: false, + output: 'line', + mode: 'standard', + print: false, + config: undefined, + }); const observability = { init: vi.fn().mockResolvedValue(undefined), decorateCall: vi.fn(() => ({})), @@ -165,6 +175,13 @@ describe('Main command observability lifecycle', () => { }); it('falls back to noop observability when initialization fails', async () => { + const envWithObservability = { + ...env, + OBSERVABILITY_BACKEND: 'langfuse', + LANGFUSE_PUBLIC_KEY: 'pk-test', + LANGFUSE_SECRET_KEY: 'sk-test', + }; + MOCK_PARSE_ENVIRONMENT.mockReturnValue(envWithObservability); const failingObservability = { init: vi.fn().mockRejectedValue(new Error('boom')), decorateCall: vi.fn(() => ({})), @@ -178,6 +195,7 @@ describe('Main command observability lifecycle', () => { await expect(program.parseAsync(['node', 'test', 'README.md'])).rejects.toThrow('process.exit:0'); + expect(MOCK_CREATE_WINSTON_LOGGER).toHaveBeenCalledWith({ level: 'info' }); expect(runtimeLogger.warn).toHaveBeenCalledWith( '[vectorlint] Observability initialization failed; continuing without telemetry', expect.objectContaining({ @@ -188,4 +206,61 @@ describe('Main command observability lifecycle', () => { const providerOptions = MOCK_CREATE_PROVIDER.mock.calls.at(-1)?.[1] as { observability?: unknown }; expect(providerOptions.observability).toBeInstanceOf(NoopObservability); }); + + it('uses a noop logger when neither verbose logging nor observability is enabled', async () => { + MOCK_PARSE_ENVIRONMENT.mockReturnValue(env); + + const { registerMainCommand } = await import('../src/cli/commands'); + const program = new Command(); + registerMainCommand(program); + + await expect(program.parseAsync(['node', 'test', 'README.md'])).rejects.toThrow('process.exit:0'); + + expect(MOCK_CREATE_WINSTON_LOGGER).not.toHaveBeenCalled(); + const providerOptions = MOCK_CREATE_PROVIDER.mock.calls.at(-1)?.[1] as { + debug?: boolean; + showPrompt?: boolean; + showPromptTrunc?: boolean; + logger?: { + debug: () => void; + info: () => void; + warn: () => void; + error: () => void; + }; + }; + + expect(providerOptions.debug).toBe(false); + expect(providerOptions.showPrompt).toBe(false); + expect(providerOptions.showPromptTrunc).toBe(false); + expect(providerOptions.logger).toBeDefined(); + expect(typeof providerOptions.logger?.debug).toBe('function'); + expect(typeof providerOptions.logger?.info).toBe('function'); + expect(typeof providerOptions.logger?.warn).toBe('function'); + expect(typeof providerOptions.logger?.error).toBe('function'); + }); + + it('creates the runtime logger when observability is enabled without verbose logging', async () => { + const envWithObservability = { + ...env, + OBSERVABILITY_BACKEND: 'langfuse', + LANGFUSE_PUBLIC_KEY: 'pk-test', + LANGFUSE_SECRET_KEY: 'sk-test', + }; + MOCK_PARSE_ENVIRONMENT.mockReturnValue(envWithObservability); + + const { registerMainCommand } = await import('../src/cli/commands'); + const program = new Command(); + registerMainCommand(program); + + await expect(program.parseAsync(['node', 'test', 'README.md'])).rejects.toThrow('process.exit:0'); + + expect(MOCK_CREATE_WINSTON_LOGGER).toHaveBeenCalledWith({ level: 'info' }); + expect(MOCK_CREATE_PROVIDER).toHaveBeenCalledWith( + envWithObservability, + expect.objectContaining({ + logger: runtimeLogger, + }), + expect.anything() + ); + }); }); diff --git a/tests/provider-factory.test.ts b/tests/provider-factory.test.ts index 328c861..08f7a7b 100644 --- a/tests/provider-factory.test.ts +++ b/tests/provider-factory.test.ts @@ -352,6 +352,21 @@ describe('Provider Factory', () => { expect(provider.config?.observability).toBe(observability); }); + + it('passes explicit provider and model names to VercelAIProvider config', () => { + const envConfig: EnvConfig = { + LLM_PROVIDER: ProviderType.OpenAI, + OPENAI_API_KEY: 'sk-test-key', + OPENAI_MODEL: 'gpt-4o', + }; + + const provider = createProvider(envConfig) as unknown as { + config?: { providerName?: string; modelName?: string }; + }; + + expect(provider.config?.providerName).toBe(ProviderType.OpenAI); + expect(provider.config?.modelName).toBe('gpt-4o'); + }); }); describe('Provider-Specific Configuration', () => { diff --git a/tests/providers/vercel-ai-provider-agent-loop.test.ts b/tests/providers/vercel-ai-provider-agent-loop.test.ts index 994aa36..b1bca66 100644 --- a/tests/providers/vercel-ai-provider-agent-loop.test.ts +++ b/tests/providers/vercel-ai-provider-agent-loop.test.ts @@ -115,6 +115,8 @@ describe('VercelAIProvider agent loop', () => { const provider = new VercelAIProvider({ model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, + providerName: 'openai', + modelName: 'gpt-4o-mini', observability, }); @@ -158,6 +160,8 @@ describe('VercelAIProvider agent loop', () => { const provider = new VercelAIProvider({ model: { provider: 'openai', modelId: 'gpt-4o-mini' } as unknown as LanguageModel, + providerName: 'openai', + modelName: 'gpt-4o-mini', logger, observability, }); diff --git a/tests/vercel-ai-provider.test.ts b/tests/vercel-ai-provider.test.ts index 605b687..ab258dd 100644 --- a/tests/vercel-ai-provider.test.ts +++ b/tests/vercel-ai-provider.test.ts @@ -235,6 +235,8 @@ describe('VercelAIProvider', () => { const provider = new VercelAIProvider({ model: MOCK_MODEL, + providerName: 'openai', + modelName: 'gpt-4o', observability, }); @@ -275,6 +277,8 @@ describe('VercelAIProvider', () => { const provider = new VercelAIProvider({ model: MOCK_MODEL, + providerName: 'openai', + modelName: 'gpt-4o', logger, observability, });