diff --git a/e2e/lib/live.ts b/e2e/lib/live.ts new file mode 100644 index 00000000..a4cd0918 --- /dev/null +++ b/e2e/lib/live.ts @@ -0,0 +1,63 @@ +import { AgentError, expect } from 'e2e'; + +// Tiny input at the lowest effort answers in seconds; OSWE-278's 429 takes about 10.5 s (one +// proxy retry). 60 s leaves headroom and stays under the runner's 120 s test timeout, so a +// stall surfaces as this request's error, not as TEST_TIMEOUT. +const BOUND_MS = 60_000; + +type Request = { body?: unknown; headers?: Record; keyless?: boolean }; + +/** + * Sends one request to CLIPROXY_LIVE_URL and classifies failures that are not the product's: + * missing inputs and a rejected key are configuration (exit 2); an unreachable proxy and a + * provider 429 or 5xx are infrastructure (exit 3). `known` accepts a failure the test asserts + * itself, matched on status and body. + */ +export async function live( + path: string, + request: Request = {}, + known?: (status: number, body: string) => boolean, +) { + const base = process.env.CLIPROXY_LIVE_URL; + if (!base) throw new AgentError('TEST_SETUP_FAILED', 'CLIPROXY_LIVE_URL is not set'); + const key = process.env.CLIPROXY_CLIENT_KEY; + if (!request.keyless && !key) { + throw new AgentError('AUTH_CREDENTIAL_UNAVAILABLE', 'CLIPROXY_CLIENT_KEY is not set'); + } + const headers: Record = { ...request.headers }; + if (!request.keyless) + Object.assign(headers, { authorization: `Bearer ${key}`, 'x-api-key': key }); + if (request.body !== undefined) headers['content-type'] = 'application/json'; + const signal = AbortSignal.timeout(BOUND_MS); + let response: Response; + try { + response = await fetch(new URL(path, base), { + method: request.body === undefined ? 'GET' : 'POST', + headers, + body: request.body === undefined ? undefined : JSON.stringify(request.body), + redirect: 'manual', + signal, + }); + } catch (cause) { + throw new AgentError('ENVIRONMENT_UNAVAILABLE', `${new URL(base).origin} unreachable`, { + cause, + }); + } + const status = response.status; + if (!request.keyless && status === 401) { + throw new AgentError('AUTH_CREDENTIAL_INVALID', `${path} rejected CLIPROXY_CLIENT_KEY (401)`); + } + if (status === 429 || status >= 500) { + const text = await response.clone().text(); + if (!known?.(status, text)) { + const excerpt = text.slice(0, 300); + throw new AgentError('ENVIRONMENT_UNAVAILABLE', `${path} answered ${status}: ${excerpt}`); + } + } + return { response, signal }; +} + +/** v7.3.17 and v8.0.13 set X-CPA-TRACE-ID once a credential is selected (cpa_trace.go). */ +export function expectTrace(response: Response) { + expect(response.headers.get('x-cpa-trace-id')).toMatch(/^\d{14}-\S+-\S+$/); +} diff --git a/e2e/tests/live.e2e.ts b/e2e/tests/live.e2e.ts new file mode 100644 index 00000000..6950f947 --- /dev/null +++ b/e2e/tests/live.e2e.ts @@ -0,0 +1,143 @@ +import { randomBytes, randomUUID } from 'node:crypto'; +import { expect, test } from 'e2e'; +import { expectTrace, live } from '../lib/live.ts'; +import { input } from '../lib/proxy.ts'; +import { readSse } from '../lib/sse.ts'; + +const onLive = { platforms: ['live'] }; +const anthropic = { 'anthropic-version': '2023-06-01' }; +const say = 'Reply with the word ok.'; + +test('live: GET /v1/models without a key returns 401', onLive, async () => { + const { response } = await live('/v1/models', { keyless: true }); + expect(response.status).toBe(401); +}); + +test('live: GET /v1/models lists Codex and Claude models', onLive, async () => { + const { response } = await live('/v1/models'); + expect(response.status).toBe(200); + const ids: string[] = (await response.json()).data.map((model: { id: string }) => model.id); + expect(ids.some((id) => id.startsWith('gpt-'))).toBe(true); + expect(ids.some((id) => id.startsWith('claude-'))).toBe(true); +}); + +test('live: Open SWE shape, non-streaming Responses on gpt-5.6-sol', onLive, async () => { + const { response } = await live('/v1/responses', { + body: { + model: 'gpt-5.6-sol', + stream: false, + store: false, + include: ['reasoning.encrypted_content'], + reasoning: { effort: 'low' }, // the lowest level the catalog lists for gpt-5.6-sol + tools: [ + { + type: 'function', + name: 'read_file', + description: 'Read a file', + parameters: { + type: 'object', + properties: { path: { type: 'string' } }, + required: ['path'], + }, + }, + ], + input: input(say), + }, + }); + expect(response.status).toBe(200); + expectTrace(response); + expect((await response.json()).status).toBe('completed'); +}); + +test('live: streaming Responses on gpt-5.6-sol ends with response.completed', onLive, async () => { + const { response, signal } = await live('/v1/responses', { + body: { + model: 'gpt-5.6-sol', + stream: true, + store: false, + reasoning: { effort: 'low' }, + input: input(say), + }, + }); + expect(response.status).toBe(200); + expectTrace(response); + const events = await readSse(response, signal); + expect(JSON.parse(events.at(-1)!.data).type).toBe('response.completed'); +}); + +test( + 'live: Claude Code shape, streaming Messages on Haiku ends with message_stop', + onLive, + async () => { + const session = randomUUID(); + // The four signals DetectClaudeCodeRequest requires to confirm a Claude Code client. + const userId = { + device_id: randomBytes(32).toString('hex'), + account_uuid: '', + session_id: session, + }; + const { response, signal } = await live('/v1/messages?beta=true', { + body: { + model: 'claude-haiku-4-5-20251001', + max_tokens: 16, + stream: true, + messages: [{ role: 'user', content: say }], + metadata: { user_id: JSON.stringify(userId) }, + }, + headers: { + ...anthropic, + 'anthropic-beta': 'claude-code-20250219', + 'user-agent': 'claude-cli/2.1.280 (external, cli)', + 'x-app': 'cli', + 'X-Claude-Code-Session-Id': session, + }, + }); + expect(response.status).toBe(200); + expectTrace(response); + const events = await readSse(response, signal); + expect(events.at(-1)?.event).toBe('message_stop'); + }, +); + +test('live: Responses on claude-sonnet-4-6 completes', onLive, async () => { + const { response } = await live('/v1/responses', { + body: { model: 'claude-sonnet-4-6', input: input(say) }, + }); + expect(response.status).toBe(200); + expectTrace(response); + expect((await response.json()).status).toBe('completed'); +}); + +// OSWE-278's response as captured from studio2. A genuine rate limit carries a descriptive +// message, so it stays an infrastructure failure (exit 3). +const isOswe278 = (status: number, body: string) => { + try { + const { error } = JSON.parse(body); + return status === 429 && error?.type === 'rate_limit_error' && error?.message === 'Error'; + } catch { + return false; + } +}; + +// Known failure: this asserts the bug. When OSWE-278 is fixed the status changes and the test +// fails visibly; then assert 200 here instead. +test( + 'live: OSWE-278 known failure, native Messages on claude-sonnet-4-6 returns 429', + onLive, + async () => { + const { response } = await live( + '/v1/messages', + { + body: { + model: 'claude-sonnet-4-6', + max_tokens: 16, + messages: [{ role: 'user', content: say }], + }, + headers: anthropic, + }, + isOswe278, + ); + expect(response.status).toBe(429); + expect((await response.json()).error).toEqual({ type: 'rate_limit_error', message: 'Error' }); + }, +);