test(e2e): live tests for the deployed proxy use cases (MASTRA-768)

e2e/lib/live.ts

@@ -0,0 +1,63 @@
+import { AgentError, expect } from 'e2e';
+
+// Tiny input at the lowest effort answers in seconds; OSWE-278's 429 takes about 10.5 s (one
+// proxy retry). 60 s leaves headroom and stays under the runner's 120 s test timeout, so a
+// stall surfaces as this request's error, not as TEST_TIMEOUT.
+const BOUND_MS = 60_000;
+
+type Request = { body?: unknown; headers?: Record<string, string>; keyless?: boolean };
+
+/**
+ * Sends one request to CLIPROXY_LIVE_URL and classifies failures that are not the product's:
+ * missing inputs and a rejected key are configuration (exit 2); an unreachable proxy and a
+ * provider 429 or 5xx are infrastructure (exit 3). `known` accepts a failure the test asserts
+ * itself, matched on status and body.
+ */
+export async function live(
+  path: string,
+  request: Request = {},
+  known?: (status: number, body: string) => boolean,
+) {
+  const base = process.env.CLIPROXY_LIVE_URL;
+  if (!base) throw new AgentError('TEST_SETUP_FAILED', 'CLIPROXY_LIVE_URL is not set');
+  const key = process.env.CLIPROXY_CLIENT_KEY;
+  if (!request.keyless && !key) {
+    throw new AgentError('AUTH_CREDENTIAL_UNAVAILABLE', 'CLIPROXY_CLIENT_KEY is not set');
+  }
+  const headers: Record<string, string> = { ...request.headers };
+  if (!request.keyless)
+    Object.assign(headers, { authorization: `Bearer ${key}`, 'x-api-key': key });
+  if (request.body !== undefined) headers['content-type'] = 'application/json';
+  const signal = AbortSignal.timeout(BOUND_MS);
+  let response: Response;
+  try {
+    response = await fetch(new URL(path, base), {
+      method: request.body === undefined ? 'GET' : 'POST',
+      headers,
+      body: request.body === undefined ? undefined : JSON.stringify(request.body),
+      redirect: 'manual',
+      signal,
+    });
+  } catch (cause) {
+    throw new AgentError('ENVIRONMENT_UNAVAILABLE', `${new URL(base).origin} unreachable`, {
+      cause,
+    });
+  }
+  const status = response.status;
+  if (!request.keyless && status === 401) {
+    throw new AgentError('AUTH_CREDENTIAL_INVALID', `${path} rejected CLIPROXY_CLIENT_KEY (401)`);
+  }
+  if (status === 429 || status >= 500) {
+    const text = await response.clone().text();
+    if (!known?.(status, text)) {
+      const excerpt = text.slice(0, 300);
+      throw new AgentError('ENVIRONMENT_UNAVAILABLE', `${path} answered ${status}: ${excerpt}`);
+    }
+  }
+  return { response, signal };
+}
+
+/** v7.3.17 and v8.0.13 set X-CPA-TRACE-ID once a credential is selected (cpa_trace.go). */
+export function expectTrace(response: Response) {
+  expect(response.headers.get('x-cpa-trace-id')).toMatch(/^\d{14}-\S+-\S+$/);
+}

e2e/tests/live.e2e.ts

@@ -0,0 +1,143 @@
+import { randomBytes, randomUUID } from 'node:crypto';
+import { expect, test } from 'e2e';
+import { expectTrace, live } from '../lib/live.ts';
+import { input } from '../lib/proxy.ts';
+import { readSse } from '../lib/sse.ts';
+
+const onLive = { platforms: ['live'] };
+const anthropic = { 'anthropic-version': '2023-06-01' };
+const say = 'Reply with the word ok.';
+
+test('live: GET /v1/models without a key returns 401', onLive, async () => {
+  const { response } = await live('/v1/models', { keyless: true });
+  expect(response.status).toBe(401);
+});
+
+test('live: GET /v1/models lists Codex and Claude models', onLive, async () => {
+  const { response } = await live('/v1/models');
+  expect(response.status).toBe(200);
+  const ids: string[] = (await response.json()).data.map((model: { id: string }) => model.id);
+  expect(ids.some((id) => id.startsWith('gpt-'))).toBe(true);
+  expect(ids.some((id) => id.startsWith('claude-'))).toBe(true);
+});
+
+test('live: Open SWE shape, non-streaming Responses on gpt-5.6-sol', onLive, async () => {
+  const { response } = await live('/v1/responses', {
+    body: {
+      model: 'gpt-5.6-sol',
+      stream: false,
+      store: false,
+      include: ['reasoning.encrypted_content'],
+      reasoning: { effort: 'low' }, // the lowest level the catalog lists for gpt-5.6-sol
+      tools: [
+        {
+          type: 'function',
+          name: 'read_file',
+          description: 'Read a file',
+          parameters: {
+            type: 'object',
+            properties: { path: { type: 'string' } },
+            required: ['path'],
+          },
+        },
+      ],
+      input: input(say),
+    },
+  });
+  expect(response.status).toBe(200);
+  expectTrace(response);
+  expect((await response.json()).status).toBe('completed');
+});
+
+test('live: streaming Responses on gpt-5.6-sol ends with response.completed', onLive, async () => {
+  const { response, signal } = await live('/v1/responses', {
+    body: {
+      model: 'gpt-5.6-sol',
+      stream: true,
+      store: false,
+      reasoning: { effort: 'low' },
+      input: input(say),
+    },
+  });
+  expect(response.status).toBe(200);
+  expectTrace(response);
+  const events = await readSse(response, signal);
+  expect(JSON.parse(events.at(-1)!.data).type).toBe('response.completed');
+});
+
+test(
+  'live: Claude Code shape, streaming Messages on Haiku ends with message_stop',
+  onLive,
+  async () => {
+    const session = randomUUID();
+    // The four signals DetectClaudeCodeRequest requires to confirm a Claude Code client.
+    const userId = {
+      device_id: randomBytes(32).toString('hex'),
+      account_uuid: '',
+      session_id: session,
+    };
+    const { response, signal } = await live('/v1/messages?beta=true', {
+      body: {
+        model: 'claude-haiku-4-5-20251001',
+        max_tokens: 16,
+        stream: true,
+        messages: [{ role: 'user', content: say }],
+        metadata: { user_id: JSON.stringify(userId) },
+      },
+      headers: {
+        ...anthropic,
+        'anthropic-beta': 'claude-code-20250219',
+        'user-agent': 'claude-cli/2.1.280 (external, cli)',
+        'x-app': 'cli',
+        'X-Claude-Code-Session-Id': session,
+      },
+    });
+    expect(response.status).toBe(200);
+    expectTrace(response);
+    const events = await readSse(response, signal);
+    expect(events.at(-1)?.event).toBe('message_stop');
+  },
+);
+
+test('live: Responses on claude-sonnet-4-6 completes', onLive, async () => {
+  const { response } = await live('/v1/responses', {
+    body: { model: 'claude-sonnet-4-6', input: input(say) },
+  });
+  expect(response.status).toBe(200);
+  expectTrace(response);
+  expect((await response.json()).status).toBe('completed');
+});
+
+// OSWE-278's response as captured from studio2. A genuine rate limit carries a descriptive
+// message, so it stays an infrastructure failure (exit 3).
+const isOswe278 = (status: number, body: string) => {
+  try {
+    const { error } = JSON.parse(body);
+    return status === 429 && error?.type === 'rate_limit_error' && error?.message === 'Error';
+  } catch {
+    return false;
+  }
+};
+
+// Known failure: this asserts the bug. When OSWE-278 is fixed the status changes and the test
+// fails visibly; then assert 200 here instead.
+test(
+  'live: OSWE-278 known failure, native Messages on claude-sonnet-4-6 returns 429',
+  onLive,
+  async () => {
+    const { response } = await live(
+      '/v1/messages',
+      {
+        body: {
+          model: 'claude-sonnet-4-6',
+          max_tokens: 16,
+          messages: [{ role: 'user', content: say }],
+        },
+        headers: anthropic,
+      },
+      isOswe278,
+    );
+    expect(response.status).toBe(429);
+    expect((await response.json()).error).toEqual({ type: 'rate_limit_error', message: 'Error' });
+  },
+);